blob: 6655f338d9e8f4713b8a103e18125a954d3cd420 [file] [log] [blame]
Jon Hall6aec96b2015-01-19 14:49:31 -08001"""
Jon Hall73cf9cc2014-11-20 22:28:38 -08002Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign mastership to controllers
8CASE3: Assign intents
9CASE4: Ping across added host intents
10CASE5: Reading state of ONOS
11CASE6: The Failure case.
12CASE7: Check state after control plane failure
13CASE8: Compare topo
14CASE9: Link s3-s28 down
15CASE10: Link s3-s28 up
16CASE11: Switch down
17CASE12: Switch up
18CASE13: Clean up
Jon Hall669173b2014-12-17 11:36:30 -080019CASE14: start election app on all onos nodes
20CASE15: Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -080021"""
Jon Hall8f89dda2015-01-22 16:03:33 -080022
23
Jon Hall73cf9cc2014-11-20 22:28:38 -080024class HATestClusterRestart:
25
Jon Hall6aec96b2015-01-19 14:49:31 -080026 def __init__( self ):
Jon Hall73cf9cc2014-11-20 22:28:38 -080027 self.default = ''
28
Jon Hall6aec96b2015-01-19 14:49:31 -080029 def CASE1( self, main ):
30 """
Jon Hall73cf9cc2014-11-20 22:28:38 -080031 CASE1 is to compile ONOS and push it to the test machines
32
33 Startup sequence:
Jon Hall73cf9cc2014-11-20 22:28:38 -080034 cell <name>
35 onos-verify-cell
36 NOTE: temporary - onos-remove-raft-logs
Jon Hall58c76b72015-02-23 11:09:24 -080037 onos-uninstall
38 start mininet
39 git pull
40 mvn clean install
41 onos-package
Jon Hall73cf9cc2014-11-20 22:28:38 -080042 onos-install -f
43 onos-wait-for-start
Jon Hall58c76b72015-02-23 11:09:24 -080044 start cli sessions
45 start tcpdump
Jon Hall6aec96b2015-01-19 14:49:31 -080046 """
47 main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
48 "initialization" )
49 main.case( "Setting up test environment" )
50 # TODO: save all the timers and output them for plotting
Jon Hall73cf9cc2014-11-20 22:28:38 -080051
52 # load some vairables from the params file
Jon Hall8f89dda2015-01-22 16:03:33 -080053 PULLCODE = False
Jon Hall6aec96b2015-01-19 14:49:31 -080054 if main.params[ 'Git' ] == 'True':
Jon Hall8f89dda2015-01-22 16:03:33 -080055 PULLCODE = True
Jon Hall529a37f2015-01-28 10:02:00 -080056 gitBranch = main.params[ 'branch' ]
Jon Hall8f89dda2015-01-22 16:03:33 -080057 cellName = main.params[ 'ENV' ][ 'cellName' ]
Jon Hall6aec96b2015-01-19 14:49:31 -080058
59 # set global variables
Jon Hall8f89dda2015-01-22 16:03:33 -080060 global ONOS1Ip
61 global ONOS1Port
62 global ONOS2Ip
63 global ONOS2Port
64 global ONOS3Ip
65 global ONOS3Port
66 global ONOS4Ip
67 global ONOS4Port
68 global ONOS5Ip
69 global ONOS5Port
70 global ONOS6Ip
71 global ONOS6Port
72 global ONOS7Ip
73 global ONOS7Port
74 global numControllers
Jon Hall73cf9cc2014-11-20 22:28:38 -080075
Jon Hall8f89dda2015-01-22 16:03:33 -080076 ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
77 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
78 ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
79 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
80 ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
81 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
82 ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
83 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
84 ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
85 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
86 ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
87 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
88 ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
89 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
90 numControllers = int( main.params[ 'num_controllers' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -080091
Jon Hall6aec96b2015-01-19 14:49:31 -080092 main.step( "Applying cell variable to environment" )
Jon Hall8f89dda2015-01-22 16:03:33 -080093 cellResult = main.ONOSbench.setCell( cellName )
94 verifyResult = main.ONOSbench.verifyCell()
Jon Hall73cf9cc2014-11-20 22:28:38 -080095
Jon Hall6aec96b2015-01-19 14:49:31 -080096 # FIXME:this is short term fix
97 main.log.report( "Removing raft logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -080098 main.ONOSbench.onosRemoveRaftLogs()
Jon Hall6aec96b2015-01-19 14:49:31 -080099 main.log.report( "Uninstalling ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800100 main.ONOSbench.onosUninstall( ONOS1Ip )
101 main.ONOSbench.onosUninstall( ONOS2Ip )
102 main.ONOSbench.onosUninstall( ONOS3Ip )
103 main.ONOSbench.onosUninstall( ONOS4Ip )
104 main.ONOSbench.onosUninstall( ONOS5Ip )
105 main.ONOSbench.onosUninstall( ONOS6Ip )
106 main.ONOSbench.onosUninstall( ONOS7Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800107
Jon Hall8f89dda2015-01-22 16:03:33 -0800108 cleanInstallResult = main.TRUE
109 gitPullResult = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800110
Jon Hall97f31752015-02-04 12:01:04 -0800111 main.step( "Starting Mininet" )
112 main.Mininet1.startNet( )
113
Jon Hall6aec96b2015-01-19 14:49:31 -0800114 main.step( "Compiling the latest version of ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800115 if PULLCODE:
Jon Hall58c76b72015-02-23 11:09:24 -0800116 main.step( "Git checkout and pull " + gitBranch )
Jon Hall529a37f2015-01-28 10:02:00 -0800117 main.ONOSbench.gitCheckout( gitBranch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800118 gitPullResult = main.ONOSbench.gitPull()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800119
Jon Hall6aec96b2015-01-19 14:49:31 -0800120 main.step( "Using mvn clean & install" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800121 cleanInstallResult = main.ONOSbench.cleanInstall()
Jon Hall6aec96b2015-01-19 14:49:31 -0800122 else:
123 main.log.warn( "Did not pull new code so skipping mvn " +
124 "clean install" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800125 main.ONOSbench.getVersion( report=True )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800126
Jon Hall6aec96b2015-01-19 14:49:31 -0800127 main.step( "Creating ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800128 packageResult = main.ONOSbench.onosPackage()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800129
Jon Hall6aec96b2015-01-19 14:49:31 -0800130 main.step( "Installing ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800131 onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
132 node=ONOS1Ip )
133 onos2InstallResult = main.ONOSbench.onosInstall( options="-f",
134 node=ONOS2Ip )
135 onos3InstallResult = main.ONOSbench.onosInstall( options="-f",
136 node=ONOS3Ip )
137 onos4InstallResult = main.ONOSbench.onosInstall( options="-f",
138 node=ONOS4Ip )
139 onos5InstallResult = main.ONOSbench.onosInstall( options="-f",
140 node=ONOS5Ip )
141 onos6InstallResult = main.ONOSbench.onosInstall( options="-f",
142 node=ONOS6Ip )
143 onos7InstallResult = main.ONOSbench.onosInstall( options="-f",
144 node=ONOS7Ip )
145 onosInstallResult = onos1InstallResult and onos2InstallResult\
146 and onos3InstallResult and onos4InstallResult\
147 and onos5InstallResult and onos6InstallResult\
148 and onos7InstallResult
Jon Hall73cf9cc2014-11-20 22:28:38 -0800149
Jon Hall6aec96b2015-01-19 14:49:31 -0800150 main.step( "Checking if ONOS is up yet" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800151 for i in range( 2 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800152 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
153 if not onos1Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800154 main.log.report( "ONOS1 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800155 main.ONOSbench.onosStop( ONOS1Ip )
156 main.ONOSbench.onosStart( ONOS1Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800157 onos2Isup = main.ONOSbench.isup( ONOS2Ip )
158 if not onos2Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800159 main.log.report( "ONOS2 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800160 main.ONOSbench.onosStop( ONOS2Ip )
161 main.ONOSbench.onosStart( ONOS2Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800162 onos3Isup = main.ONOSbench.isup( ONOS3Ip )
163 if not onos3Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800164 main.log.report( "ONOS3 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800165 main.ONOSbench.onosStop( ONOS3Ip )
166 main.ONOSbench.onosStart( ONOS3Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800167 onos4Isup = main.ONOSbench.isup( ONOS4Ip )
168 if not onos4Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800169 main.log.report( "ONOS4 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800170 main.ONOSbench.onosStop( ONOS4Ip )
171 main.ONOSbench.onosStart( ONOS4Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800172 onos5Isup = main.ONOSbench.isup( ONOS5Ip )
173 if not onos5Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800174 main.log.report( "ONOS5 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800175 main.ONOSbench.onosStop( ONOS5Ip )
176 main.ONOSbench.onosStart( ONOS5Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800177 onos6Isup = main.ONOSbench.isup( ONOS6Ip )
178 if not onos6Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800179 main.log.report( "ONOS6 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800180 main.ONOSbench.onosStop( ONOS6Ip )
181 main.ONOSbench.onosStart( ONOS6Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800182 onos7Isup = main.ONOSbench.isup( ONOS7Ip )
183 if not onos7Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800184 main.log.report( "ONOS7 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800185 main.ONOSbench.onosStop( ONOS7Ip )
186 main.ONOSbench.onosStart( ONOS7Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800187 onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
188 and onos4Isup and onos5Isup and onos6Isup and onos7Isup
189 if onosIsupResult == main.TRUE:
Jon Hall94fd0472014-12-08 11:52:42 -0800190 break
Jon Hall73cf9cc2014-11-20 22:28:38 -0800191
Jon Hall8f89dda2015-01-22 16:03:33 -0800192 cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
193 cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
194 cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
195 cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
196 cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
197 cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
198 cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
199 cliResults = cliResult1 and cliResult2 and cliResult3 and\
200 cliResult4 and cliResult5 and cliResult6 and cliResult7
Jon Hall73cf9cc2014-11-20 22:28:38 -0800201
Jon Hall6aec96b2015-01-19 14:49:31 -0800202 main.step( "Start Packet Capture MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800203 main.Mininet2.startTcpdump(
Jon Hall6aec96b2015-01-19 14:49:31 -0800204 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
205 + "-MN.pcap",
206 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
207 port=main.params[ 'MNtcpdump' ][ 'port' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800208
Jon Hall8f89dda2015-01-22 16:03:33 -0800209 case1Result = ( cleanInstallResult and packageResult and
210 cellResult and verifyResult and onosInstallResult
211 and onosIsupResult and cliResults )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800212
Jon Hall8f89dda2015-01-22 16:03:33 -0800213 utilities.assert_equals( expect=main.TRUE, actual=case1Result,
Jon Hall58c76b72015-02-23 11:09:24 -0800214 onpass="Test startup successful",
215 onfail="Test startup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800216
Jon Hall8f89dda2015-01-22 16:03:33 -0800217 if case1Result == main.FALSE:
Jon Hall94fd0472014-12-08 11:52:42 -0800218 main.cleanup()
219 main.exit()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800220
Jon Hall6aec96b2015-01-19 14:49:31 -0800221 def CASE2( self, main ):
222 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800223 Assign mastership to controllers
Jon Hall6aec96b2015-01-19 14:49:31 -0800224 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800225 import re
226
Jon Hall6aec96b2015-01-19 14:49:31 -0800227 main.log.report( "Assigning switches to controllers" )
228 main.case( "Assigning Controllers" )
229 main.step( "Assign switches to controllers" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800230
Jon Hall6aec96b2015-01-19 14:49:31 -0800231 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800232 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -0800233 sw=str( i ),
Jon Hall8f89dda2015-01-22 16:03:33 -0800234 count=numControllers,
235 ip1=ONOS1Ip, port1=ONOS1Port,
236 ip2=ONOS2Ip, port2=ONOS2Port,
237 ip3=ONOS3Ip, port3=ONOS3Port,
238 ip4=ONOS4Ip, port4=ONOS4Port,
239 ip5=ONOS5Ip, port5=ONOS5Port,
240 ip6=ONOS6Ip, port6=ONOS6Port,
241 ip7=ONOS7Ip, port7=ONOS7Port )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800242
Jon Hall8f89dda2015-01-22 16:03:33 -0800243 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800244 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800245 response = main.Mininet1.getSwController( "s" + str( i ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800246 try:
Jon Hall6aec96b2015-01-19 14:49:31 -0800247 main.log.info( str( response ) )
Jon Hallfebb1c72015-03-05 13:30:09 -0800248 except Exception:
Jon Hall6aec96b2015-01-19 14:49:31 -0800249 main.log.info( repr( response ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800250 if re.search( "tcp:" + ONOS1Ip, response )\
251 and re.search( "tcp:" + ONOS2Ip, response )\
252 and re.search( "tcp:" + ONOS3Ip, response )\
253 and re.search( "tcp:" + ONOS4Ip, response )\
254 and re.search( "tcp:" + ONOS5Ip, response )\
255 and re.search( "tcp:" + ONOS6Ip, response )\
256 and re.search( "tcp:" + ONOS7Ip, response ):
257 mastershipCheck = mastershipCheck and main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800258 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800259 mastershipCheck = main.FALSE
260 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800261 main.log.report( "Switch mastership assigned correctly" )
262 utilities.assert_equals(
263 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800264 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800265 onpass="Switch mastership assigned correctly",
266 onfail="Switches not assigned correctly to controllers" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800267
Jon Hall6aec96b2015-01-19 14:49:31 -0800268 # Manually assign mastership to the controller we want
Jon Hall8f89dda2015-01-22 16:03:33 -0800269 roleCall = main.TRUE
270 roleCheck = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -0800271 try:
272 # Assign switch
273 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
274 assert deviceId, "No device id for s1 in ONOS"
Jon Hall8f89dda2015-01-22 16:03:33 -0800275 roleCall = roleCall and main.ONOScli1.deviceRole(
276 deviceId,
Jon Hall58c76b72015-02-23 11:09:24 -0800277 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800278 # Check assignment
Jon Hall58c76b72015-02-23 11:09:24 -0800279 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800280 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800281 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800282 roleCheck = roleCheck and main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800283
Jon Hall58c76b72015-02-23 11:09:24 -0800284 # Assign switch
285 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
286 assert deviceId, "No device id for s28 in ONOS"
Jon Hall8f89dda2015-01-22 16:03:33 -0800287 roleCall = roleCall and main.ONOScli1.deviceRole(
288 deviceId,
Jon Hall58c76b72015-02-23 11:09:24 -0800289 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800290 # Check assignment
Jon Hall58c76b72015-02-23 11:09:24 -0800291 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800292 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800293 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800294 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800295
Jon Hall58c76b72015-02-23 11:09:24 -0800296 # Assign switch
297 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
298 assert deviceId, "No device id for s2 in ONOS"
299 roleCall = roleCall and main.ONOScli1.deviceRole(
300 deviceId,
301 ONOS2Ip )
302 # Check assignment
303 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
304 roleCheck = roleCheck and main.TRUE
305 else:
306 roleCheck = roleCheck and main.FALSE
307
308 # Assign switch
309 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
310 assert deviceId, "No device id for s3 in ONOS"
311 roleCall = roleCall and main.ONOScli1.deviceRole(
312 deviceId,
313 ONOS2Ip )
314 # Check assignment
315 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
316 roleCheck = roleCheck and main.TRUE
317 else:
318 roleCheck = roleCheck and main.FALSE
319
320 # Assign switch
321 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
322 assert deviceId, "No device id for s5 in ONOS"
323 roleCall = roleCall and main.ONOScli1.deviceRole(
324 deviceId,
325 ONOS3Ip )
326 # Check assignment
327 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
328 roleCheck = roleCheck and main.TRUE
329 else:
330 roleCheck = roleCheck and main.FALSE
331
332 # Assign switch
333 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
334 assert deviceId, "No device id for s6 in ONOS"
335 roleCall = roleCall and main.ONOScli1.deviceRole(
336 deviceId,
337 ONOS3Ip )
338 # Check assignment
339 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
340 roleCheck = roleCheck and main.TRUE
341 else:
342 roleCheck = roleCheck and main.FALSE
343
344 # Assign switch
345 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
346 assert deviceId, "No device id for s4 in ONOS"
347 roleCall = roleCall and main.ONOScli1.deviceRole(
348 deviceId,
349 ONOS4Ip )
350 # Check assignment
351 if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
352 roleCheck = roleCheck and main.TRUE
353 else:
354 roleCheck = roleCheck and main.FALSE
355
356 for i in range( 8, 18 ):
357 dpid = '3' + str( i ).zfill( 3 )
358 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
359 assert deviceId, "No device id for s%i in ONOS" % i
360 roleCall = roleCall and main.ONOScli1.deviceRole(
361 deviceId,
362 ONOS5Ip )
363 # Check assignment
364 if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
365 roleCheck = roleCheck and main.TRUE
366 else:
367 roleCheck = roleCheck and main.FALSE
368
369 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
370 assert deviceId, "No device id for s7 in ONOS"
371 roleCall = roleCall and main.ONOScli1.deviceRole(
372 deviceId,
373 ONOS6Ip )
374 # Check assignment
375 if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
376 roleCheck = roleCheck and main.TRUE
377 else:
378 roleCheck = roleCheck and main.FALSE
379
380 for i in range( 18, 28 ):
381 dpid = '6' + str( i ).zfill( 3 )
382 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
383 assert deviceId, "No device id for s%i in ONOS" % i
384 roleCall = roleCall and main.ONOScli1.deviceRole(
385 deviceId,
386 ONOS7Ip )
387 # Check assignment
388 if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
389 roleCheck = roleCheck and main.TRUE
390 else:
391 roleCheck = roleCheck and main.FALSE
392 except ( AttributeError, AssertionError ):
393 main.log.exception( "Something is wrong with ONOS device view" )
394 main.log.info( main.ONOScli1.devices() )
395
Jon Hall6aec96b2015-01-19 14:49:31 -0800396 utilities.assert_equals(
397 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800398 actual=roleCall,
Jon Hall6aec96b2015-01-19 14:49:31 -0800399 onpass="Re-assigned switch mastership to designated controller",
Jon Hall8f89dda2015-01-22 16:03:33 -0800400 onfail="Something wrong with deviceRole calls" )
Jon Hall94fd0472014-12-08 11:52:42 -0800401
Jon Hall6aec96b2015-01-19 14:49:31 -0800402 utilities.assert_equals(
403 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800404 actual=roleCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800405 onpass="Switches were successfully reassigned to designated " +
406 "controller",
407 onfail="Switches were not successfully reassigned" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800408 mastershipCheck = mastershipCheck and roleCall and roleCheck
409 utilities.assert_equals( expect=main.TRUE, actual=mastershipCheck,
Jon Hall21270ac2015-02-16 17:59:55 -0800410 onpass="Switch mastership correctly assigned",
411 onfail="Error in (re)assigning switch" +
412 " mastership" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800413
Jon Hall6aec96b2015-01-19 14:49:31 -0800414 def CASE3( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800415 """
416 Assign intents
Jon Hall73cf9cc2014-11-20 22:28:38 -0800417 """
Jon Hall6aec96b2015-01-19 14:49:31 -0800418 # FIXME: we must reinstall intents until we have a persistant
419 # datastore!
Jon Hall73cf9cc2014-11-20 22:28:38 -0800420 import time
Jon Hall58c76b72015-02-23 11:09:24 -0800421 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800422 main.log.report( "Adding host intents" )
423 main.case( "Adding host Intents" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800424
Jon Hall8f89dda2015-01-22 16:03:33 -0800425 main.step( "Discovering Hosts( Via pingall for now )" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800426 # FIXME: Once we have a host discovery mechanism, use that instead
Jon Hall73cf9cc2014-11-20 22:28:38 -0800427
Jon Hall6aec96b2015-01-19 14:49:31 -0800428 # install onos-app-fwd
429 main.log.info( "Install reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800430 main.ONOScli1.featureInstall( "onos-app-fwd" )
431 main.ONOScli2.featureInstall( "onos-app-fwd" )
432 main.ONOScli3.featureInstall( "onos-app-fwd" )
433 main.ONOScli4.featureInstall( "onos-app-fwd" )
434 main.ONOScli5.featureInstall( "onos-app-fwd" )
435 main.ONOScli6.featureInstall( "onos-app-fwd" )
436 main.ONOScli7.featureInstall( "onos-app-fwd" )
Jon Hall94fd0472014-12-08 11:52:42 -0800437
Jon Hall6aec96b2015-01-19 14:49:31 -0800438 # REACTIVE FWD test
Jon Hall8f89dda2015-01-22 16:03:33 -0800439 pingResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800440 time1 = time.time()
Jon Hall8f89dda2015-01-22 16:03:33 -0800441 pingResult = main.Mininet1.pingall()
Jon Hall529a37f2015-01-28 10:02:00 -0800442 utilities.assert_equals(
443 expect=main.TRUE,
444 actual=pingResult,
445 onpass="Reactive Pingall test passed",
446 onfail="Reactive Pingall failed, one or more ping pairs failed" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800447 time2 = time.time()
Jon Hall6aec96b2015-01-19 14:49:31 -0800448 main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800449
Jon Hall6aec96b2015-01-19 14:49:31 -0800450 # uninstall onos-app-fwd
451 main.log.info( "Uninstall reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800452 main.ONOScli1.featureUninstall( "onos-app-fwd" )
453 main.ONOScli2.featureUninstall( "onos-app-fwd" )
454 main.ONOScli3.featureUninstall( "onos-app-fwd" )
455 main.ONOScli4.featureUninstall( "onos-app-fwd" )
456 main.ONOScli5.featureUninstall( "onos-app-fwd" )
457 main.ONOScli6.featureUninstall( "onos-app-fwd" )
458 main.ONOScli7.featureUninstall( "onos-app-fwd" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800459 # timeout for fwd flows
460 time.sleep( 10 )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800461
Jon Hall6aec96b2015-01-19 14:49:31 -0800462 main.step( "Add host intents" )
Jon Hall58c76b72015-02-23 11:09:24 -0800463 intentIds = []
Jon Hall6aec96b2015-01-19 14:49:31 -0800464 # TODO: move the host numbers to params
Jon Hall58c76b72015-02-23 11:09:24 -0800465 # Maybe look at all the paths we ping?
Jon Hall8f89dda2015-01-22 16:03:33 -0800466 intentAddResult = True
Jon Hall58c76b72015-02-23 11:09:24 -0800467 hostResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800468 for i in range( 8, 18 ):
469 main.log.info( "Adding host intent between h" + str( i ) +
470 " and h" + str( i + 10 ) )
471 host1 = "00:00:00:00:00:" + \
472 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
473 host2 = "00:00:00:00:00:" + \
474 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800475 # NOTE: getHost can return None
476 host1Dict = main.ONOScli1.getHost( host1 )
477 host2Dict = main.ONOScli1.getHost( host2 )
478 host1Id = None
479 host2Id = None
480 if host1Dict and host2Dict:
481 host1Id = host1Dict.get( 'id', None )
482 host2Id = host2Dict.get( 'id', None )
Jon Hall8f89dda2015-01-22 16:03:33 -0800483 if host1Id and host2Id:
Jon Hall21270ac2015-02-16 17:59:55 -0800484
Jon Hall58c76b72015-02-23 11:09:24 -0800485 tmpId = main.ONOScli1.addHostIntent(
Jon Hall8f89dda2015-01-22 16:03:33 -0800486 host1Id,
487 host2Id )
Jon Hall63604932015-02-26 17:09:50 -0800488 if tmpId:
489 main.log.info( "Added intent with id: " + tmpId )
490 intentIds.append( tmpId )
491 else:
492 main.log.error( "addHostIntent reutrned None" )
Jon Hall669173b2014-12-17 11:36:30 -0800493 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800494 main.log.error( "Error, getHost() failed" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800495 main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
496 sort_keys=True,
497 indent=4,
498 separators=( ',', ': ' ) ) )
Jon Hall58c76b72015-02-23 11:09:24 -0800499 hostResult = main.FALSE
500 onosIds = main.ONOScli1.getAllIntentsId()
501 main.log.info( "Submitted intents: " + str( intentIds ) )
502 main.log.info( "Intents in ONOS: " + str( onosIds ) )
503 for intent in intentIds:
504 if intent in onosIds:
505 pass # intent submitted is still in onos
506 else:
507 intentAddResult = False
Jon Hall1b8f54a2015-02-04 13:24:20 -0800508 # Print the intent states
Jon Hall58c76b72015-02-23 11:09:24 -0800509 intents = main.ONOScli1.intents()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800510 intentStates = []
Jon Hall63604932015-02-26 17:09:50 -0800511 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800512 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
513 count = 0
Jon Hall1b8f54a2015-02-04 13:24:20 -0800514 for intent in json.loads( intents ): # Iter through intents of a node
Jon Hall58c76b72015-02-23 11:09:24 -0800515 state = intent.get( 'state', None )
Jon Hall63604932015-02-26 17:09:50 -0800516 if "INSTALLED" not in state:
517 installedCheck = False
Jon Hall58c76b72015-02-23 11:09:24 -0800518 intentId = intent.get( 'id', None )
519 intentStates.append( ( intentId, state ) )
520 # add submitted intents not in the store
521 tmplist = [ i for i, s in intentStates ]
522 missingIntents = False
523 for i in intentIds:
524 if i not in tmplist:
525 intentStates.append( ( i, " - " ) )
526 missingIntents = True
527 intentStates.sort()
528 for i, s in intentStates:
529 count += 1
530 main.log.info( "%-6s%-15s%-15s" %
531 ( str( count ), str( i ), str( s ) ) )
Jon Hall63604932015-02-26 17:09:50 -0800532 main.ONOScli1.leaders()
533 main.ONOScli1.partitions()
534 # for node in nodes:
535 # node.pendingMap()
536 pendingMap = main.ONOScli1.pendingMap()
537 main.ONOScli2.pendingMap()
538 main.ONOScli3.pendingMap()
539 main.ONOScli4.pendingMap()
540 main.ONOScli5.pendingMap()
541 main.ONOScli6.pendingMap()
542 main.ONOScli7.pendingMap()
Jon Hall58c76b72015-02-23 11:09:24 -0800543 intentAddResult = bool( pingResult and hostResult and intentAddResult
Jon Hall63604932015-02-26 17:09:50 -0800544 and not missingIntents and installedCheck )
Jon Hall6aec96b2015-01-19 14:49:31 -0800545 utilities.assert_equals(
546 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800547 actual=intentAddResult,
Jon Hall529a37f2015-01-28 10:02:00 -0800548 onpass="Pushed host intents to ONOS",
549 onfail="Error in pushing host intents to ONOS" )
Jon Hall58c76b72015-02-23 11:09:24 -0800550
Jon Hall63604932015-02-26 17:09:50 -0800551 if not intentAddResult or "key" in pendingMap:
Jon Hall58c76b72015-02-23 11:09:24 -0800552 import time
Jon Hall63604932015-02-26 17:09:50 -0800553 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800554 main.log.info( "Sleeping 60 seconds to see if intents are found" )
555 time.sleep( 60 )
556 onosIds = main.ONOScli1.getAllIntentsId()
557 main.log.info( "Submitted intents: " + str( intentIds ) )
558 main.log.info( "Intents in ONOS: " + str( onosIds ) )
559 # Print the intent states
560 intents = main.ONOScli1.intents()
561 intentStates = []
562 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
563 count = 0
564 for intent in json.loads( intents ):
565 # Iter through intents of a node
566 state = intent.get( 'state', None )
Jon Hall63604932015-02-26 17:09:50 -0800567 if "INSTALLED" not in state:
568 installedCheck = False
Jon Hall58c76b72015-02-23 11:09:24 -0800569 intentId = intent.get( 'id', None )
570 intentStates.append( ( intentId, state ) )
571 # add submitted intents not in the store
572 tmplist = [ i for i, s in intentStates ]
573 for i in intentIds:
574 if i not in tmplist:
575 intentStates.append( ( i, " - " ) )
576 intentStates.sort()
577 for i, s in intentStates:
578 count += 1
579 main.log.info( "%-6s%-15s%-15s" %
580 ( str( count ), str( i ), str( s ) ) )
Jon Hall63604932015-02-26 17:09:50 -0800581 main.ONOScli1.leaders()
582 main.ONOScli1.pendingMap()
583 main.ONOScli2.pendingMap()
584 main.ONOScli3.pendingMap()
585 main.ONOScli4.pendingMap()
586 main.ONOScli5.pendingMap()
587 main.ONOScli6.pendingMap()
588 main.ONOScli7.pendingMap()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800589
Jon Hall6aec96b2015-01-19 14:49:31 -0800590 def CASE4( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800591 """
592 Ping across added host intents
593 """
Jon Hall58c76b72015-02-23 11:09:24 -0800594 import json
Jon Hall73cf9cc2014-11-20 22:28:38 -0800595 description = " Ping across added host intents"
Jon Hall6aec96b2015-01-19 14:49:31 -0800596 main.log.report( description )
597 main.case( description )
Jon Hall8f89dda2015-01-22 16:03:33 -0800598 PingResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800599 for i in range( 8, 18 ):
Jon Hall58c76b72015-02-23 11:09:24 -0800600 ping = main.Mininet1.pingHost( src="h" + str( i ),
601 target="h" + str( i + 10 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800602 PingResult = PingResult and ping
Jon Hall6aec96b2015-01-19 14:49:31 -0800603 if ping == main.FALSE:
604 main.log.warn( "Ping failed between h" + str( i ) +
605 " and h" + str( i + 10 ) )
606 elif ping == main.TRUE:
607 main.log.info( "Ping test passed!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800608 # Don't set PingResult or you'd override failures
Jon Hall8f89dda2015-01-22 16:03:33 -0800609 if PingResult == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800610 main.log.report(
611 "Intents have not been installed correctly, pings failed." )
Jon Hall58c76b72015-02-23 11:09:24 -0800612 # TODO: pretty print
Jon Hall529a37f2015-01-28 10:02:00 -0800613 main.log.warn( "ONSO1 intents: " )
614 main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
615 sort_keys=True,
616 indent=4,
617 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800618 if PingResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800619 main.log.report(
620 "Intents have been installed correctly and verified by pings" )
621 utilities.assert_equals(
622 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800623 actual=PingResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800624 onpass="Intents have been installed correctly and pings work",
625 onfail="Intents have not been installed correctly, pings failed." )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800626
Jon Hall63604932015-02-26 17:09:50 -0800627 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800628 if PingResult is not main.TRUE:
629 # Print the intent states
630 intents = main.ONOScli1.intents()
631 intentStates = []
632 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
633 count = 0
634 # Iter through intents of a node
635 for intent in json.loads( intents ):
636 state = intent.get( 'state', None )
Jon Hall63604932015-02-26 17:09:50 -0800637 if "INSTALLED" not in state:
638 installedCheck = False
Jon Hall58c76b72015-02-23 11:09:24 -0800639 intentId = intent.get( 'id', None )
640 intentStates.append( ( intentId, state ) )
641 intentStates.sort()
642 for i, s in intentStates:
643 count += 1
644 main.log.info( "%-6s%-15s%-15s" %
645 ( str( count ), str( i ), str( s ) ) )
Jon Hall63604932015-02-26 17:09:50 -0800646 main.ONOScli1.leaders()
647 main.ONOScli1.partitions()
648 main.ONOScli1.pendingMap()
649 main.ONOScli2.pendingMap()
650 main.ONOScli3.pendingMap()
651 main.ONOScli4.pendingMap()
652 main.ONOScli5.pendingMap()
653 main.ONOScli6.pendingMap()
654 main.ONOScli7.pendingMap()
655 if not installedCheck:
656 main.log.info( "Waiting 60 seconds to see if intent states change" )
657 time.sleep( 60 )
658 # Print the intent states
659 intents = main.ONOScli1.intents()
660 intentStates = []
661 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
662 count = 0
663 # Iter through intents of a node
664 for intent in json.loads( intents ):
665 state = intent.get( 'state', None )
666 if "INSTALLED" not in state:
667 installedCheck = False
668 intentId = intent.get( 'id', None )
669 intentStates.append( ( intentId, state ) )
670 intentStates.sort()
671 for i, s in intentStates:
672 count += 1
673 main.log.info( "%-6s%-15s%-15s" %
674 ( str( count ), str( i ), str( s ) ) )
675 main.ONOScli1.leaders()
676 main.ONOScli1.partitions()
677 main.ONOScli1.pendingMap()
678 main.ONOScli2.pendingMap()
679 main.ONOScli3.pendingMap()
680 main.ONOScli4.pendingMap()
681 main.ONOScli5.pendingMap()
682 main.ONOScli6.pendingMap()
683 main.ONOScli7.pendingMap()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800684
Jon Hall6aec96b2015-01-19 14:49:31 -0800685 def CASE5( self, main ):
686 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800687 Reading state of ONOS
Jon Hall6aec96b2015-01-19 14:49:31 -0800688 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800689 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800690 # assumes that sts is already in you PYTHONPATH
691 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -0800692
Jon Hall6aec96b2015-01-19 14:49:31 -0800693 main.log.report( "Setting up and gathering data for current state" )
694 main.case( "Setting up and gathering data for current state" )
695 # The general idea for this test case is to pull the state of
696 # ( intents,flows, topology,... ) from each ONOS node
697 # We can then compare them with eachother and also with past states
Jon Hall73cf9cc2014-11-20 22:28:38 -0800698
Jon Hall6aec96b2015-01-19 14:49:31 -0800699 main.step( "Get the Mastership of each switch from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800700 global mastershipState
701 mastershipState = []
Jon Hall94fd0472014-12-08 11:52:42 -0800702
Jon Hall6aec96b2015-01-19 14:49:31 -0800703 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -0800704 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
705 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
706 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
707 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
708 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
709 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
710 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
711 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
712 ONOS3MasterNotNull and ONOS4MasterNotNull and\
713 ONOS5MasterNotNull and ONOS6MasterNotNull and\
714 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -0800715 utilities.assert_equals(
716 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800717 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -0800718 onpass="Each device has a master",
719 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -0800720
Jon Hall8f89dda2015-01-22 16:03:33 -0800721 ONOS1Mastership = main.ONOScli1.roles()
722 ONOS2Mastership = main.ONOScli2.roles()
723 ONOS3Mastership = main.ONOScli3.roles()
724 ONOS4Mastership = main.ONOScli4.roles()
725 ONOS5Mastership = main.ONOScli5.roles()
726 ONOS6Mastership = main.ONOScli6.roles()
727 ONOS7Mastership = main.ONOScli7.roles()
728 if "Error" in ONOS1Mastership or not ONOS1Mastership\
729 or "Error" in ONOS2Mastership or not ONOS2Mastership\
730 or "Error" in ONOS3Mastership or not ONOS3Mastership\
731 or "Error" in ONOS4Mastership or not ONOS4Mastership\
732 or "Error" in ONOS5Mastership or not ONOS5Mastership\
733 or "Error" in ONOS6Mastership or not ONOS6Mastership\
734 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -0800735 main.log.report( "Error in getting ONOS roles" )
736 main.log.warn(
737 "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800738 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800739 main.log.warn(
740 "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800741 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800742 main.log.warn(
743 "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800744 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800745 main.log.warn(
746 "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800747 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800748 main.log.warn(
749 "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800750 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800751 main.log.warn(
752 "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800753 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800754 main.log.warn(
755 "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800756 repr( ONOS7Mastership ) )
757 consistentMastership = main.FALSE
758 elif ONOS1Mastership == ONOS2Mastership\
759 and ONOS1Mastership == ONOS3Mastership\
760 and ONOS1Mastership == ONOS4Mastership\
761 and ONOS1Mastership == ONOS5Mastership\
762 and ONOS1Mastership == ONOS6Mastership\
763 and ONOS1Mastership == ONOS7Mastership:
764 mastershipState = ONOS1Mastership
765 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800766 main.log.report(
767 "Switch roles are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800768 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800769 main.log.warn(
770 "ONOS1 roles: ",
771 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800772 json.loads( ONOS1Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800773 sort_keys=True,
774 indent=4,
775 separators=(
776 ',',
777 ': ' ) ) )
778 main.log.warn(
779 "ONOS2 roles: ",
780 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800781 json.loads( ONOS2Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800782 sort_keys=True,
783 indent=4,
784 separators=(
785 ',',
786 ': ' ) ) )
787 main.log.warn(
788 "ONOS3 roles: ",
789 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800790 json.loads( ONOS3Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800791 sort_keys=True,
792 indent=4,
793 separators=(
794 ',',
795 ': ' ) ) )
796 main.log.warn(
797 "ONOS4 roles: ",
798 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800799 json.loads( ONOS4Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800800 sort_keys=True,
801 indent=4,
802 separators=(
803 ',',
804 ': ' ) ) )
805 main.log.warn(
806 "ONOS5 roles: ",
807 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800808 json.loads( ONOS5Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800809 sort_keys=True,
810 indent=4,
811 separators=(
812 ',',
813 ': ' ) ) )
814 main.log.warn(
815 "ONOS6 roles: ",
816 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800817 json.loads( ONOS6Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800818 sort_keys=True,
819 indent=4,
820 separators=(
821 ',',
822 ': ' ) ) )
823 main.log.warn(
824 "ONOS7 roles: ",
825 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800826 json.loads( ONOS7Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800827 sort_keys=True,
828 indent=4,
829 separators=(
830 ',',
831 ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800832 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800833 utilities.assert_equals(
834 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800835 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -0800836 onpass="Switch roles are consistent across all ONOS nodes",
837 onfail="ONOS nodes have different views of switch roles" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800838
Jon Hall6aec96b2015-01-19 14:49:31 -0800839 main.step( "Get the intents from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800840 global intentState
841 intentState = []
842 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
843 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
844 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
845 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
846 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
847 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
848 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
849 intentCheck = main.FALSE
850 if "Error" in ONOS1Intents or not ONOS1Intents\
851 or "Error" in ONOS2Intents or not ONOS2Intents\
852 or "Error" in ONOS3Intents or not ONOS3Intents\
853 or "Error" in ONOS4Intents or not ONOS4Intents\
854 or "Error" in ONOS5Intents or not ONOS5Intents\
855 or "Error" in ONOS6Intents or not ONOS6Intents\
856 or "Error" in ONOS7Intents or not ONOS7Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -0800857 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800858 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
859 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
860 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
861 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
862 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
863 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
864 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
865 elif ONOS1Intents == ONOS2Intents\
866 and ONOS1Intents == ONOS3Intents\
867 and ONOS1Intents == ONOS4Intents\
868 and ONOS1Intents == ONOS5Intents\
869 and ONOS1Intents == ONOS6Intents\
870 and ONOS1Intents == ONOS7Intents:
871 intentState = ONOS1Intents
872 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800873 main.log.report( "Intents are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800874 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800875 main.log.warn(
876 "ONOS1 intents: ",
877 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800878 json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800879 sort_keys=True,
880 indent=4,
881 separators=(
882 ',',
883 ': ' ) ) )
884 main.log.warn(
885 "ONOS2 intents: ",
886 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800887 json.loads( ONOS2Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800888 sort_keys=True,
889 indent=4,
890 separators=(
891 ',',
892 ': ' ) ) )
893 main.log.warn(
894 "ONOS3 intents: ",
895 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800896 json.loads( ONOS3Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800897 sort_keys=True,
898 indent=4,
899 separators=(
900 ',',
901 ': ' ) ) )
902 main.log.warn(
903 "ONOS4 intents: ",
904 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800905 json.loads( ONOS4Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800906 sort_keys=True,
907 indent=4,
908 separators=(
909 ',',
910 ': ' ) ) )
911 main.log.warn(
912 "ONOS5 intents: ",
913 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800914 json.loads( ONOS5Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800915 sort_keys=True,
916 indent=4,
917 separators=(
918 ',',
919 ': ' ) ) )
920 main.log.warn(
921 "ONOS6 intents: ",
922 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800923 json.loads( ONOS6Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800924 sort_keys=True,
925 indent=4,
926 separators=(
927 ',',
928 ': ' ) ) )
929 main.log.warn(
930 "ONOS7 intents: ",
931 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800932 json.loads( ONOS7Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800933 sort_keys=True,
934 indent=4,
935 separators=(
936 ',',
937 ': ' ) ) )
938 utilities.assert_equals(
939 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800940 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800941 onpass="Intents are consistent across all ONOS nodes",
942 onfail="ONOS nodes have different views of intents" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800943
Jon Hall6aec96b2015-01-19 14:49:31 -0800944 main.step( "Get the flows from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800945 global flowState
946 flowState = []
Jon Hall8f89dda2015-01-22 16:03:33 -0800947 flowCheck = main.FALSE
Jon Hall58c76b72015-02-23 11:09:24 -0800948 try:
949 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
950 ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
951 ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
952 ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
953 ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
954 ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
955 ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
956 assert ONOS1Flows, "ONOS1 Flows should not be empty"
957 assert ONOS2Flows, "ONOS2 Flows should not be empty"
958 assert ONOS3Flows, "ONOS3 Flows should not be empty"
959 assert ONOS4Flows, "ONOS4 Flows should not be empty"
960 assert ONOS5Flows, "ONOS5 Flows should not be empty"
961 assert ONOS6Flows, "ONOS6 Flows should not be empty"
962 assert ONOS7Flows, "ONOS7 Flows should not be empty"
963 assert "Error" not in ONOS1Flows, "ONOS1 Flows contains 'Error'"
964 assert "Error" not in ONOS2Flows, "ONOS2 Flows contains 'Error'"
965 assert "Error" not in ONOS3Flows, "ONOS3 Flows contains 'Error'"
966 assert "Error" not in ONOS4Flows, "ONOS4 Flows contains 'Error'"
967 assert "Error" not in ONOS5Flows, "ONOS5 Flows contains 'Error'"
968 assert "Error" not in ONOS6Flows, "ONOS6 Flows contains 'Error'"
969 assert "Error" not in ONOS7Flows, "ONOS7 Flows contains 'Error'"
970 ONOS1FlowsJson = json.loads( ONOS1Flows )
971 ONOS2FlowsJson = json.loads( ONOS2Flows )
972 ONOS3FlowsJson = json.loads( ONOS3Flows )
973 ONOS4FlowsJson = json.loads( ONOS4Flows )
974 ONOS5FlowsJson = json.loads( ONOS5Flows )
975 ONOS6FlowsJson = json.loads( ONOS6Flows )
976 ONOS7FlowsJson = json.loads( ONOS7Flows )
977 except ( ValueError, AssertionError ): # From json.loads, or asserts
978 main.log.exception( "One or more 'flows' responses from " +
979 "ONOS couldn't be decoded." )
Jon Hall8f89dda2015-01-22 16:03:33 -0800980 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
981 main.log.warn( "ONOS2 flows repsponse: " + ONOS2Flows )
982 main.log.warn( "ONOS3 flows repsponse: " + ONOS3Flows )
983 main.log.warn( "ONOS4 flows repsponse: " + ONOS4Flows )
984 main.log.warn( "ONOS5 flows repsponse: " + ONOS5Flows )
985 main.log.warn( "ONOS6 flows repsponse: " + ONOS6Flows )
986 main.log.warn( "ONOS7 flows repsponse: " + ONOS7Flows )
Jon Hall58c76b72015-02-23 11:09:24 -0800987 else: # No exceptions
988 if len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
989 and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
990 and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
991 and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
992 and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
993 and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
Jon Hall6aec96b2015-01-19 14:49:31 -0800994 # TODO: Do a better check, maybe compare flows on switches?
Jon Hall58c76b72015-02-23 11:09:24 -0800995 # NOTE Possible issue with this not always being set?
996 flowState = ONOS1Flows
997 flowCheck = main.TRUE
998 main.log.report( "Flow count is consistent across all" +
999 " ONOS nodes" )
1000 else:
1001 main.log.warn( "ONOS1 flows: " +
1002 json.dumps( ONOS1FlowsJson, sort_keys=True,
1003 indent=4,
1004 separators=( ',', ': ' ) ) )
1005 main.log.warn( "ONOS2 flows: " +
1006 json.dumps( ONOS2FlowsJson, sort_keys=True,
1007 indent=4,
1008 separators=( ',', ': ' ) ) )
1009 main.log.warn( "ONOS3 flows: " +
1010 json.dumps( ONOS3FlowsJson, sort_keys=True,
1011 indent=4,
1012 separators=( ',', ': ' ) ) )
1013 main.log.warn( "ONOS4 flows: " +
1014 json.dumps( ONOS4FlowsJson, sort_keys=True,
1015 indent=4,
1016 separators=( ',', ': ' ) ) )
1017 main.log.warn( "ONOS5 flows: " +
1018 json.dumps( ONOS5FlowsJson, sort_keys=True,
1019 indent=4,
1020 separators=( ',', ': ' ) ) )
1021 main.log.warn( "ONOS6 flows: " +
1022 json.dumps( ONOS6FlowsJson, sort_keys=True,
1023 indent=4,
1024 separators=( ',', ': ' ) ) )
1025 main.log.warn( "ONOS7 flows: " +
1026 json.dumps( ONOS7FlowsJson, sort_keys=True,
1027 indent=4,
1028 separators=( ',', ': ' ) ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001029 utilities.assert_equals(
1030 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001031 actual=flowCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001032 onpass="The flow count is consistent across all ONOS nodes",
1033 onfail="ONOS nodes have different flow counts" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001034
Jon Hall6aec96b2015-01-19 14:49:31 -08001035 main.step( "Get the OF Table entries" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001036 global flows
Jon Hall6aec96b2015-01-19 14:49:31 -08001037 flows = []
1038 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001039 flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001040 if flowCheck == main.FALSE:
1041 for table in flows:
1042 main.log.warn( table )
Jon Hall6aec96b2015-01-19 14:49:31 -08001043 # TODO: Compare switch flow tables with ONOS flow tables
Jon Hall73cf9cc2014-11-20 22:28:38 -08001044
Jon Hall6aec96b2015-01-19 14:49:31 -08001045 main.step( "Start continuous pings" )
1046 main.Mininet2.pingLong(
1047 src=main.params[ 'PING' ][ 'source1' ],
1048 target=main.params[ 'PING' ][ 'target1' ],
1049 pingTime=500 )
1050 main.Mininet2.pingLong(
1051 src=main.params[ 'PING' ][ 'source2' ],
1052 target=main.params[ 'PING' ][ 'target2' ],
1053 pingTime=500 )
1054 main.Mininet2.pingLong(
1055 src=main.params[ 'PING' ][ 'source3' ],
1056 target=main.params[ 'PING' ][ 'target3' ],
1057 pingTime=500 )
1058 main.Mininet2.pingLong(
1059 src=main.params[ 'PING' ][ 'source4' ],
1060 target=main.params[ 'PING' ][ 'target4' ],
1061 pingTime=500 )
1062 main.Mininet2.pingLong(
1063 src=main.params[ 'PING' ][ 'source5' ],
1064 target=main.params[ 'PING' ][ 'target5' ],
1065 pingTime=500 )
1066 main.Mininet2.pingLong(
1067 src=main.params[ 'PING' ][ 'source6' ],
1068 target=main.params[ 'PING' ][ 'target6' ],
1069 pingTime=500 )
1070 main.Mininet2.pingLong(
1071 src=main.params[ 'PING' ][ 'source7' ],
1072 target=main.params[ 'PING' ][ 'target7' ],
1073 pingTime=500 )
1074 main.Mininet2.pingLong(
1075 src=main.params[ 'PING' ][ 'source8' ],
1076 target=main.params[ 'PING' ][ 'target8' ],
1077 pingTime=500 )
1078 main.Mininet2.pingLong(
1079 src=main.params[ 'PING' ][ 'source9' ],
1080 target=main.params[ 'PING' ][ 'target9' ],
1081 pingTime=500 )
1082 main.Mininet2.pingLong(
1083 src=main.params[ 'PING' ][ 'source10' ],
1084 target=main.params[ 'PING' ][ 'target10' ],
1085 pingTime=500 )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001086
Jon Hall6aec96b2015-01-19 14:49:31 -08001087 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001088 ctrls = []
1089 count = 1
1090 while True:
1091 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -08001092 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
1093 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
1094 temp = temp + ( "ONOS" + str( count ), )
1095 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
1096 temp = temp + \
1097 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
1098 ctrls.append( temp )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001099 count = count + 1
1100 else:
1101 break
Jon Hall6aec96b2015-01-19 14:49:31 -08001102 MNTopo = TestONTopology(
1103 main.Mininet1,
1104 ctrls ) # can also add Intent API info for intent operations
Jon Hall73cf9cc2014-11-20 22:28:38 -08001105
Jon Hall6aec96b2015-01-19 14:49:31 -08001106 main.step( "Collecting topology information from ONOS" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001107 devices = []
1108 devices.append( main.ONOScli1.devices() )
1109 devices.append( main.ONOScli2.devices() )
1110 devices.append( main.ONOScli3.devices() )
1111 devices.append( main.ONOScli4.devices() )
1112 devices.append( main.ONOScli5.devices() )
1113 devices.append( main.ONOScli6.devices() )
1114 devices.append( main.ONOScli7.devices() )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001115 hosts = []
Jon Hall58c76b72015-02-23 11:09:24 -08001116 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1117 hosts.append( json.loads( main.ONOScli2.hosts() ) )
1118 hosts.append( json.loads( main.ONOScli3.hosts() ) )
1119 hosts.append( json.loads( main.ONOScli4.hosts() ) )
1120 hosts.append( json.loads( main.ONOScli5.hosts() ) )
1121 hosts.append( json.loads( main.ONOScli6.hosts() ) )
1122 hosts.append( json.loads( main.ONOScli7.hosts() ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001123 ports = []
1124 ports.append( main.ONOScli1.ports() )
1125 ports.append( main.ONOScli2.ports() )
1126 ports.append( main.ONOScli3.ports() )
1127 ports.append( main.ONOScli4.ports() )
1128 ports.append( main.ONOScli5.ports() )
1129 ports.append( main.ONOScli6.ports() )
1130 ports.append( main.ONOScli7.ports() )
1131 links = []
1132 links.append( main.ONOScli1.links() )
1133 links.append( main.ONOScli2.links() )
1134 links.append( main.ONOScli3.links() )
1135 links.append( main.ONOScli4.links() )
1136 links.append( main.ONOScli5.links() )
1137 links.append( main.ONOScli6.links() )
1138 links.append( main.ONOScli7.links() )
Jon Hall94fd0472014-12-08 11:52:42 -08001139 clusters = []
1140 clusters.append( main.ONOScli1.clusters() )
1141 clusters.append( main.ONOScli2.clusters() )
1142 clusters.append( main.ONOScli3.clusters() )
1143 clusters.append( main.ONOScli4.clusters() )
1144 clusters.append( main.ONOScli5.clusters() )
1145 clusters.append( main.ONOScli6.clusters() )
1146 clusters.append( main.ONOScli7.clusters() )
Jon Hall529a37f2015-01-28 10:02:00 -08001147 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001148
Jon Hall6aec96b2015-01-19 14:49:31 -08001149 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -08001150 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001151 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001152 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001153 if "Error" not in hosts[ controller ]:
1154 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001155 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001156 else: # hosts not consistent
1157 main.log.report( "hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001158 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001159 " is inconsistent with ONOS1" )
1160 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001161 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001162
1163 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001164 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001165 controllerStr )
1166 consistentHostsResult = main.FALSE
1167 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001168 " hosts response: " +
1169 repr( hosts[ controller ] ) )
1170 utilities.assert_equals(
1171 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001172 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001173 onpass="Hosts view is consistent across all ONOS nodes",
1174 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001175
Jon Hall58c76b72015-02-23 11:09:24 -08001176 ipResult = main.TRUE
1177 for controller in range( 0, len( hosts ) ):
1178 controllerStr = str( controller + 1 )
1179 for host in hosts[ controller ]:
1180 if host.get( 'ips', [] ) == []:
1181 main.log.error(
1182 "DEBUG:Error with host ips on controller" +
1183 controllerStr + ": " + str( host ) )
1184 ipResult = main.FALSE
1185 utilities.assert_equals(
1186 expect=main.TRUE,
1187 actual=ipResult,
1188 onpass="The ips of the hosts aren't empty",
1189 onfail="The ip of at least one host is missing" )
1190
Jon Hall6aec96b2015-01-19 14:49:31 -08001191 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001192 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001193 for controller in range( len( clusters ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001194 if "Error" not in clusters[ controller ]:
1195 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001196 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001197 else: # clusters not consistent
1198 main.log.report( "clusters from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001199 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001200 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001201 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001202
1203 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001204 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001205 "from ONOS" + controllerStr )
1206 consistentClustersResult = main.FALSE
1207 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001208 " clusters response: " +
1209 repr( clusters[ controller ] ) )
1210 utilities.assert_equals(
1211 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001212 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001213 onpass="Clusters view is consistent across all ONOS nodes",
1214 onfail="ONOS nodes have different views of clusters" )
1215 # there should always only be one cluster
Jon Hall8f89dda2015-01-22 16:03:33 -08001216 numClusters = len( json.loads( clusters[ 0 ] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001217 clusterResults = main.FALSE
1218 if numClusters == 1:
1219 clusterResults = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001220 utilities.assert_equals(
1221 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001222 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001223 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08001224 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001225
Jon Hall6aec96b2015-01-19 14:49:31 -08001226 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001227 devicesResults = main.TRUE
1228 portsResults = main.TRUE
1229 linksResults = main.TRUE
1230 for controller in range( numControllers ):
1231 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001232 if devices[ controller ] or "Error" not in devices[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001233 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001234 MNTopo,
1235 json.loads(
1236 devices[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001237 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001238 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001239 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001240 actual=currentDevicesResult,
1241 onpass="ONOS" + controllerStr +
1242 " Switches view is correct",
1243 onfail="ONOS" + controllerStr +
1244 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001245
Jon Hall6aec96b2015-01-19 14:49:31 -08001246 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001247 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001248 MNTopo,
1249 json.loads(
1250 ports[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001251 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001252 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001253 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001254 actual=currentPortsResult,
1255 onpass="ONOS" + controllerStr +
1256 " ports view is correct",
1257 onfail="ONOS" + controllerStr +
1258 " ports view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001259
Jon Hall6aec96b2015-01-19 14:49:31 -08001260 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001261 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001262 MNTopo,
1263 json.loads(
1264 links[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001265 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001266 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001267 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001268 actual=currentLinksResult,
1269 onpass="ONOS" + controllerStr +
1270 " links view is correct",
1271 onfail="ONOS" + controllerStr +
1272 " links view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001273
Jon Hall8f89dda2015-01-22 16:03:33 -08001274 devicesResults = devicesResults and currentDevicesResult
1275 portsResults = portsResults and currentPortsResult
1276 linksResults = linksResults and currentLinksResult
Jon Hall73cf9cc2014-11-20 22:28:38 -08001277
Jon Hall8f89dda2015-01-22 16:03:33 -08001278 topoResult = devicesResults and portsResults and linksResults\
Jon Hall58c76b72015-02-23 11:09:24 -08001279 and consistentHostsResult and consistentClustersResult\
1280 and clusterResults and ipResult
Jon Hall8f89dda2015-01-22 16:03:33 -08001281 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08001282 onpass="Topology Check Test successful",
1283 onfail="Topology Check Test NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001284
Jon Hall8f89dda2015-01-22 16:03:33 -08001285 finalAssert = main.TRUE
1286 finalAssert = finalAssert and topoResult and flowCheck \
Jon Hall58c76b72015-02-23 11:09:24 -08001287 and intentCheck and consistentMastership and rolesNotNull
Jon Hall8f89dda2015-01-22 16:03:33 -08001288 utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
Jon Hall58c76b72015-02-23 11:09:24 -08001289 onpass="State check successful",
1290 onfail="State check NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001291
Jon Hall6aec96b2015-01-19 14:49:31 -08001292 def CASE6( self, main ):
1293 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001294 The Failure case.
Jon Hall6aec96b2015-01-19 14:49:31 -08001295 """
1296 main.log.report( "Restart entire ONOS cluster" )
1297 main.log.case( "Restart entire ONOS cluster" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001298 main.ONOSbench.onosKill( ONOS1Ip )
1299 main.ONOSbench.onosKill( ONOS2Ip )
1300 main.ONOSbench.onosKill( ONOS3Ip )
1301 main.ONOSbench.onosKill( ONOS4Ip )
1302 main.ONOSbench.onosKill( ONOS5Ip )
1303 main.ONOSbench.onosKill( ONOS6Ip )
1304 main.ONOSbench.onosKill( ONOS7Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001305
Jon Hall6aec96b2015-01-19 14:49:31 -08001306 main.step( "Checking if ONOS is up yet" )
Jon Hallffb386d2014-11-21 13:43:38 -08001307 count = 0
Jon Hall8f89dda2015-01-22 16:03:33 -08001308 onosIsupResult = main.FALSE
1309 while onosIsupResult == main.FALSE and count < 10:
1310 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
1311 onos2Isup = main.ONOSbench.isup( ONOS2Ip )
1312 onos3Isup = main.ONOSbench.isup( ONOS3Ip )
1313 onos4Isup = main.ONOSbench.isup( ONOS4Ip )
1314 onos5Isup = main.ONOSbench.isup( ONOS5Ip )
1315 onos6Isup = main.ONOSbench.isup( ONOS6Ip )
1316 onos7Isup = main.ONOSbench.isup( ONOS7Ip )
1317 onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
1318 and onos4Isup and onos5Isup and onos6Isup and onos7Isup
Jon Hallffb386d2014-11-21 13:43:38 -08001319 count = count + 1
Jon Hall73cf9cc2014-11-20 22:28:38 -08001320 # TODO: if it becomes an issue, we can retry this step a few times
1321
Jon Hall8f89dda2015-01-22 16:03:33 -08001322 cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
1323 cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
1324 cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
1325 cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
1326 cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
1327 cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
1328 cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
1329 cliResults = cliResult1 and cliResult2 and cliResult3\
1330 and cliResult4 and cliResult5 and cliResult6\
1331 and cliResult7
Jon Hall73cf9cc2014-11-20 22:28:38 -08001332
Jon Hall8f89dda2015-01-22 16:03:33 -08001333 caseResults = main.TRUE and onosIsupResult and cliResults
1334 utilities.assert_equals( expect=main.TRUE, actual=caseResults,
Jon Hall58c76b72015-02-23 11:09:24 -08001335 onpass="ONOS restart successful",
1336 onfail="ONOS restart NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001337
Jon Hall6aec96b2015-01-19 14:49:31 -08001338 def CASE7( self, main ):
1339 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001340 Check state after ONOS failure
Jon Hall6aec96b2015-01-19 14:49:31 -08001341 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001342 import json
Jon Hall6aec96b2015-01-19 14:49:31 -08001343 main.case( "Running ONOS Constant State Tests" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001344
Jon Hall6aec96b2015-01-19 14:49:31 -08001345 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -08001346 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
1347 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
1348 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
1349 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
1350 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
1351 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
1352 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
1353 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
1354 ONOS3MasterNotNull and ONOS4MasterNotNull and\
1355 ONOS5MasterNotNull and ONOS6MasterNotNull and\
1356 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -08001357 utilities.assert_equals(
1358 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001359 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -08001360 onpass="Each device has a master",
1361 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -08001362
Jon Hall6aec96b2015-01-19 14:49:31 -08001363 main.step( "Check if switch roles are consistent across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001364 ONOS1Mastership = main.ONOScli1.roles()
1365 ONOS2Mastership = main.ONOScli2.roles()
1366 ONOS3Mastership = main.ONOScli3.roles()
1367 ONOS4Mastership = main.ONOScli4.roles()
1368 ONOS5Mastership = main.ONOScli5.roles()
1369 ONOS6Mastership = main.ONOScli6.roles()
1370 ONOS7Mastership = main.ONOScli7.roles()
1371 if "Error" in ONOS1Mastership or not ONOS1Mastership\
1372 or "Error" in ONOS2Mastership or not ONOS2Mastership\
1373 or "Error" in ONOS3Mastership or not ONOS3Mastership\
1374 or "Error" in ONOS4Mastership or not ONOS4Mastership\
1375 or "Error" in ONOS5Mastership or not ONOS5Mastership\
1376 or "Error" in ONOS6Mastership or not ONOS6Mastership\
1377 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -08001378 main.log.error( "Error in getting ONOS mastership" )
1379 main.log.warn( "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001380 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001381 main.log.warn( "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001382 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001383 main.log.warn( "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001384 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001385 main.log.warn( "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001386 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001387 main.log.warn( "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001388 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001389 main.log.warn( "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001390 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001391 main.log.warn( "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001392 repr( ONOS7Mastership ) )
1393 consistentMastership = main.FALSE
1394 elif ONOS1Mastership == ONOS2Mastership\
1395 and ONOS1Mastership == ONOS3Mastership\
1396 and ONOS1Mastership == ONOS4Mastership\
1397 and ONOS1Mastership == ONOS5Mastership\
1398 and ONOS1Mastership == ONOS6Mastership\
1399 and ONOS1Mastership == ONOS7Mastership:
1400 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001401 main.log.report(
1402 "Switch roles are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001403 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001404 main.log.warn( "ONOS1 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001405 json.loads( ONOS1Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001406 separators=( ',', ': ' ) ) )
1407 main.log.warn( "ONOS2 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001408 json.loads( ONOS2Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001409 separators=( ',', ': ' ) ) )
1410 main.log.warn( "ONOS3 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001411 json.loads( ONOS3Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001412 separators=( ',', ': ' ) ) )
1413 main.log.warn( "ONOS4 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001414 json.loads( ONOS4Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001415 separators=( ',', ': ' ) ) )
1416 main.log.warn( "ONOS5 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001417 json.loads( ONOS5Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001418 separators=( ',', ': ' ) ) )
1419 main.log.warn( "ONOS6 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001420 json.loads( ONOS6Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001421 separators=( ',', ': ' ) ) )
1422 main.log.warn( "ONOS7 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001423 json.loads( ONOS7Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001424 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001425 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001426 utilities.assert_equals(
1427 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001428 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -08001429 onpass="Switch roles are consistent across all ONOS nodes",
1430 onfail="ONOS nodes have different views of switch roles" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001431
1432 description2 = "Compare switch roles from before failure"
Jon Hall6aec96b2015-01-19 14:49:31 -08001433 main.step( description2 )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001434
Jon Hall8f89dda2015-01-22 16:03:33 -08001435 currentJson = json.loads( ONOS1Mastership )
1436 oldJson = json.loads( mastershipState )
1437 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001438 for i in range( 1, 29 ):
1439 switchDPID = str(
1440 main.Mininet1.getSwitchDPID(
1441 switch="s" +
1442 str( i ) ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001443
Jon Hall8f89dda2015-01-22 16:03:33 -08001444 current = [ switch[ 'master' ] for switch in currentJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001445 if switchDPID in switch[ 'id' ] ]
Jon Hall8f89dda2015-01-22 16:03:33 -08001446 old = [ switch[ 'master' ] for switch in oldJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001447 if switchDPID in switch[ 'id' ] ]
Jon Hall73cf9cc2014-11-20 22:28:38 -08001448 if current == old:
Jon Hall8f89dda2015-01-22 16:03:33 -08001449 mastershipCheck = mastershipCheck and main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001450 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001451 main.log.warn( "Mastership of switch %s changed" % switchDPID )
Jon Hall8f89dda2015-01-22 16:03:33 -08001452 mastershipCheck = main.FALSE
1453 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001454 main.log.report( "Mastership of Switches was not changed" )
1455 utilities.assert_equals(
1456 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001457 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001458 onpass="Mastership of Switches was not changed",
1459 onfail="Mastership of some switches changed" )
1460 # NOTE: we expect mastership to change on controller failure
Jon Hall8f89dda2015-01-22 16:03:33 -08001461 mastershipCheck = mastershipCheck and consistentMastership
Jon Hall73cf9cc2014-11-20 22:28:38 -08001462
Jon Hall6aec96b2015-01-19 14:49:31 -08001463 main.step( "Get the intents and compare across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001464 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1465 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
1466 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
1467 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
1468 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
1469 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
1470 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
1471 intentCheck = main.FALSE
1472 if "Error" in ONOS1Intents or not ONOS1Intents\
1473 or "Error" in ONOS2Intents or not ONOS2Intents\
1474 or "Error" in ONOS3Intents or not ONOS3Intents\
1475 or "Error" in ONOS4Intents or not ONOS4Intents\
1476 or "Error" in ONOS5Intents or not ONOS5Intents\
1477 or "Error" in ONOS6Intents or not ONOS6Intents\
1478 or "Error" in ONOS7Intents or not ONOS7Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -08001479 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001480 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1481 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
1482 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
1483 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
1484 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
1485 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
1486 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
1487 elif ONOS1Intents == ONOS2Intents\
1488 and ONOS1Intents == ONOS3Intents\
1489 and ONOS1Intents == ONOS4Intents\
1490 and ONOS1Intents == ONOS5Intents\
1491 and ONOS1Intents == ONOS6Intents\
1492 and ONOS1Intents == ONOS7Intents:
1493 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001494 main.log.report( "Intents are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001495 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001496 main.log.warn( "ONOS1 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001497 print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001498 indent=4, separators=( ',', ': ' ) )
1499 main.log.warn( "ONOS2 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001500 print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001501 indent=4, separators=( ',', ': ' ) )
1502 main.log.warn( "ONOS3 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001503 print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001504 indent=4, separators=( ',', ': ' ) )
1505 main.log.warn( "ONOS4 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001506 print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001507 indent=4, separators=( ',', ': ' ) )
1508 main.log.warn( "ONOS5 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001509 print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001510 indent=4, separators=( ',', ': ' ) )
1511 main.log.warn( "ONOS6 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001512 print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001513 indent=4, separators=( ',', ': ' ) )
1514 main.log.warn( "ONOS7 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001515 print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001516 indent=4, separators=( ',', ': ' ) )
1517 utilities.assert_equals(
1518 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001519 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001520 onpass="Intents are consistent across all ONOS nodes",
1521 onfail="ONOS nodes have different views of intents" )
Jon Hall1b8f54a2015-02-04 13:24:20 -08001522 # Print the intent states
1523 intents = []
1524 intents.append( ONOS1Intents )
1525 intents.append( ONOS2Intents )
1526 intents.append( ONOS3Intents )
1527 intents.append( ONOS4Intents )
1528 intents.append( ONOS5Intents )
1529 intents.append( ONOS6Intents )
1530 intents.append( ONOS7Intents )
1531 intentStates = []
1532 for node in intents: # Iter through ONOS nodes
1533 nodeStates = []
Jon Hall58c76b72015-02-23 11:09:24 -08001534 # Iter through intents of a node
1535 for intent in json.loads( node ):
Jon Hall1b8f54a2015-02-04 13:24:20 -08001536 nodeStates.append( intent[ 'state' ] )
1537 intentStates.append( nodeStates )
1538 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1539 main.log.info( dict( out ) )
1540
Jon Hall58c76b72015-02-23 11:09:24 -08001541 # NOTE: Store has no durability, so intents are lost across system
1542 # restarts
Jon Hall6aec96b2015-01-19 14:49:31 -08001543 """
1544 main.step( "Compare current intents with intents before the failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001545 # NOTE: this requires case 5 to pass for intentState to be set.
Jon Hall94fd0472014-12-08 11:52:42 -08001546 # maybe we should stop the test if that fails?
Jon Hall1b8f54a2015-02-04 13:24:20 -08001547 sameIntents = main.TRUE
1548 if intentState and intentState == ONOS1Intents:
Jon Hall8f89dda2015-01-22 16:03:33 -08001549 sameIntents = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001550 main.log.report( "Intents are consistent with before failure" )
1551 # TODO: possibly the states have changed? we may need to figure out
1552 # what the aceptable states are
Jon Hall73cf9cc2014-11-20 22:28:38 -08001553 else:
Jon Hall669173b2014-12-17 11:36:30 -08001554 try:
Jon Hall6aec96b2015-01-19 14:49:31 -08001555 main.log.warn( "ONOS1 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001556 print json.dumps( json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -08001557 sort_keys=True, indent=4,
1558 separators=( ',', ': ' ) )
Jon Hallfebb1c72015-03-05 13:30:09 -08001559 except Exception:
Jon Hall669173b2014-12-17 11:36:30 -08001560 pass
Jon Hall8f89dda2015-01-22 16:03:33 -08001561 sameIntents = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001562 utilities.assert_equals(
1563 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001564 actual=sameIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001565 onpass="Intents are consistent with before failure",
1566 onfail="The Intents changed during failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001567 intentCheck = intentCheck and sameIntents
Jon Hall6aec96b2015-01-19 14:49:31 -08001568 """
1569 main.step( "Get the OF Table entries and compare to before " +
1570 "component failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001571 FlowTables = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001572 flows2 = []
1573 for i in range( 28 ):
1574 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001575 tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
1576 flows2.append( tmpFlows )
1577 tempResult = main.Mininet2.flowComp(
Jon Hall6aec96b2015-01-19 14:49:31 -08001578 flow1=flows[ i ],
Jon Hall8f89dda2015-01-22 16:03:33 -08001579 flow2=tmpFlows )
1580 FlowTables = FlowTables and tempResult
1581 if FlowTables == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001582 main.log.info( "Differences in flow table for switch: s" +
1583 str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001584 if FlowTables == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001585 main.log.report( "No changes were found in the flow tables" )
1586 utilities.assert_equals(
1587 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001588 actual=FlowTables,
Jon Hall6aec96b2015-01-19 14:49:31 -08001589 onpass="No changes were found in the flow tables",
1590 onfail="Changes were found in the flow tables" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001591
Jon Hall6aec96b2015-01-19 14:49:31 -08001592 main.step( "Check the continuous pings to ensure that no packets " +
1593 "were dropped during component failure" )
1594 # FIXME: This check is always failing. Investigate cause
1595 # NOTE: this may be something to do with file permsissions
Jon Hall73cf9cc2014-11-20 22:28:38 -08001596 # or slight change in format
Jon Hall6aec96b2015-01-19 14:49:31 -08001597 main.Mininet2.pingKill(
1598 main.params[ 'TESTONUSER' ],
1599 main.params[ 'TESTONIP' ] )
Jon Hall8f89dda2015-01-22 16:03:33 -08001600 LossInPings = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001601 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1602 for i in range( 8, 18 ):
1603 main.log.info(
1604 "Checking for a loss in pings along flow from s" +
1605 str( i ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001606 LossInPings = main.Mininet2.checkForLoss(
Jon Hall6aec96b2015-01-19 14:49:31 -08001607 "/tmp/ping.h" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001608 str( i ) ) or LossInPings
1609 if LossInPings == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001610 main.log.info( "Loss in ping detected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001611 elif LossInPings == main.ERROR:
Jon Hall6aec96b2015-01-19 14:49:31 -08001612 main.log.info( "There are multiple mininet process running" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001613 elif LossInPings == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001614 main.log.info( "No Loss in the pings" )
1615 main.log.report( "No loss of dataplane connectivity" )
1616 utilities.assert_equals(
1617 expect=main.FALSE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001618 actual=LossInPings,
Jon Hall6aec96b2015-01-19 14:49:31 -08001619 onpass="No Loss of connectivity",
1620 onfail="Loss of dataplane connectivity detected" )
Jon Hall58c76b72015-02-23 11:09:24 -08001621 # NOTE: Since intents are not persisted with IntnentStore,
1622 # we expect loss in dataplane connectivity
Jon Hall8f89dda2015-01-22 16:03:33 -08001623 LossInPings = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001624
Jon Hall6aec96b2015-01-19 14:49:31 -08001625 # Test of LeadershipElection
Jon Hall8f89dda2015-01-22 16:03:33 -08001626 leaderList = []
1627 leaderResult = main.TRUE
1628 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001629 # loop through ONOScli handlers
1630 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001631 leaderN = node.electionTestLeader()
1632 leaderList.append( leaderN )
Jon Hall669173b2014-12-17 11:36:30 -08001633 if leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001634 # error in response
1635 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001636 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001637 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001638 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001639 elif leaderN is None:
1640 main.log.report( "ONOS" + str( controller ) +
1641 " shows no leader for the election-app was" +
1642 " elected after the old one died" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001643 leaderResult = main.FALSE
1644 if len( set( leaderList ) ) != 1:
1645 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001646 main.log.error(
1647 "Inconsistent view of leader for the election test app" )
1648 # TODO: print the list
Jon Hall8f89dda2015-01-22 16:03:33 -08001649 if leaderResult:
1650 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001651 "view of leader across listeners and a new " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001652 "leader was re-elected if applicable )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001653 utilities.assert_equals(
1654 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001655 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001656 onpass="Leadership election passed",
1657 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08001658
Jon Hall8f89dda2015-01-22 16:03:33 -08001659 result = ( mastershipCheck and intentCheck and FlowTables and
1660 ( not LossInPings ) and rolesNotNull and leaderResult )
Jon Hall6aec96b2015-01-19 14:49:31 -08001661 result = int( result )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001662 if result == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001663 main.log.report( "Constant State Tests Passed" )
1664 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08001665 onpass="Constant State Tests Passed",
1666 onfail="Constant state tests failed" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001667
Jon Hall6aec96b2015-01-19 14:49:31 -08001668 def CASE8( self, main ):
1669 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001670 Compare topo
Jon Hall6aec96b2015-01-19 14:49:31 -08001671 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001672 import sys
Jon Hall6aec96b2015-01-19 14:49:31 -08001673 # FIXME add this path to params
1674 sys.path.append( "/home/admin/sts" )
1675 # assumes that sts is already in you PYTHONPATH
1676 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -08001677 import json
1678 import time
1679
Jon Hall6aec96b2015-01-19 14:49:31 -08001680 description = "Compare ONOS Topology view to Mininet topology"
1681 main.case( description )
1682 main.log.report( description )
1683 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001684 ctrls = []
1685 count = 1
1686 while True:
1687 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -08001688 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
1689 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
1690 temp = temp + ( "ONOS" + str( count ), )
1691 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
1692 temp = temp + \
1693 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
1694 ctrls.append( temp )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001695 count = count + 1
1696 else:
1697 break
Jon Hall6aec96b2015-01-19 14:49:31 -08001698 MNTopo = TestONTopology(
1699 main.Mininet1,
1700 ctrls ) # can also add Intent API info for intent operations
Jon Hall73cf9cc2014-11-20 22:28:38 -08001701
Jon Hall6aec96b2015-01-19 14:49:31 -08001702 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001703 devicesResults = main.TRUE
1704 portsResults = main.TRUE
1705 linksResults = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -08001706 hostsResults = main.TRUE
Jon Hall8f89dda2015-01-22 16:03:33 -08001707 topoResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001708 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -08001709 count = 0
Jon Hall6aec96b2015-01-19 14:49:31 -08001710 main.step( "Collecting topology information from ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001711 startTime = time.time()
Jon Hall21270ac2015-02-16 17:59:55 -08001712 # Give time for Gossip to work
Jon Hall8f89dda2015-01-22 16:03:33 -08001713 while topoResult == main.FALSE and elapsed < 60:
Jon Hallffb386d2014-11-21 13:43:38 -08001714 count = count + 1
Jon Hall94fd0472014-12-08 11:52:42 -08001715 if count > 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08001716 # TODO: Depricate STS usage
Jon Hall58c76b72015-02-23 11:09:24 -08001717 MNTopo = TestONTopology( main.Mininet1, ctrls )
Jon Hall8f89dda2015-01-22 16:03:33 -08001718 cliStart = time.time()
Jon Hall94fd0472014-12-08 11:52:42 -08001719 devices = []
1720 devices.append( main.ONOScli1.devices() )
1721 devices.append( main.ONOScli2.devices() )
1722 devices.append( main.ONOScli3.devices() )
1723 devices.append( main.ONOScli4.devices() )
1724 devices.append( main.ONOScli5.devices() )
1725 devices.append( main.ONOScli6.devices() )
1726 devices.append( main.ONOScli7.devices() )
1727 hosts = []
Jon Hall529a37f2015-01-28 10:02:00 -08001728 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1729 hosts.append( json.loads( main.ONOScli2.hosts() ) )
1730 hosts.append( json.loads( main.ONOScli3.hosts() ) )
1731 hosts.append( json.loads( main.ONOScli4.hosts() ) )
1732 hosts.append( json.loads( main.ONOScli5.hosts() ) )
1733 hosts.append( json.loads( main.ONOScli6.hosts() ) )
1734 hosts.append( json.loads( main.ONOScli7.hosts() ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001735 ipResult = main.TRUE
Jon Hall529a37f2015-01-28 10:02:00 -08001736 for controller in range( 0, len( hosts ) ):
1737 controllerStr = str( controller + 1 )
1738 for host in hosts[ controller ]:
Jon Hall58c76b72015-02-23 11:09:24 -08001739 if host is None or host.get( 'ips', [] ) == []:
Jon Hall529a37f2015-01-28 10:02:00 -08001740 main.log.error(
1741 "DEBUG:Error with host ips on controller" +
1742 controllerStr + ": " + str( host ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001743 ipResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001744 ports = []
1745 ports.append( main.ONOScli1.ports() )
1746 ports.append( main.ONOScli2.ports() )
1747 ports.append( main.ONOScli3.ports() )
1748 ports.append( main.ONOScli4.ports() )
1749 ports.append( main.ONOScli5.ports() )
1750 ports.append( main.ONOScli6.ports() )
1751 ports.append( main.ONOScli7.ports() )
1752 links = []
1753 links.append( main.ONOScli1.links() )
1754 links.append( main.ONOScli2.links() )
1755 links.append( main.ONOScli3.links() )
1756 links.append( main.ONOScli4.links() )
1757 links.append( main.ONOScli5.links() )
1758 links.append( main.ONOScli6.links() )
1759 links.append( main.ONOScli7.links() )
1760 clusters = []
1761 clusters.append( main.ONOScli1.clusters() )
1762 clusters.append( main.ONOScli2.clusters() )
1763 clusters.append( main.ONOScli3.clusters() )
1764 clusters.append( main.ONOScli4.clusters() )
1765 clusters.append( main.ONOScli5.clusters() )
1766 clusters.append( main.ONOScli6.clusters() )
1767 clusters.append( main.ONOScli7.clusters() )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001768
Jon Hall8f89dda2015-01-22 16:03:33 -08001769 elapsed = time.time() - startTime
1770 cliTime = time.time() - cliStart
1771 print "CLI time: " + str( cliTime )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001772
Jon Hall8f89dda2015-01-22 16:03:33 -08001773 for controller in range( numControllers ):
1774 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001775 if devices[ controller ] or "Error" not in devices[
1776 controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001777 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001778 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001779 json.loads( devices[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001780 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001781 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001782 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001783 actual=currentDevicesResult,
1784 onpass="ONOS" + controllerStr +
1785 " Switches view is correct",
1786 onfail="ONOS" + controllerStr +
1787 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001788
Jon Hall6aec96b2015-01-19 14:49:31 -08001789 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001790 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001791 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001792 json.loads( ports[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001793 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001794 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001795 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001796 actual=currentPortsResult,
1797 onpass="ONOS" + controllerStr +
1798 " ports view is correct",
1799 onfail="ONOS" + controllerStr +
1800 " ports view is incorrect" )
Jon Hall94fd0472014-12-08 11:52:42 -08001801
Jon Hall6aec96b2015-01-19 14:49:31 -08001802 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001803 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001804 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001805 json.loads( links[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001806 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001807 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001808 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001809 actual=currentLinksResult,
1810 onpass="ONOS" + controllerStr +
1811 " links view is correct",
1812 onfail="ONOS" + controllerStr +
1813 " links view is incorrect" )
1814
1815 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1816 currentHostsResult = main.Mininet1.compareHosts(
1817 MNTopo, hosts[ controller ] )
1818 else:
1819 currentHostsResult = main.FALSE
1820 utilities.assert_equals( expect=main.TRUE,
1821 actual=currentHostsResult,
1822 onpass="ONOS" + controllerStr +
1823 " hosts exist in Mininet",
1824 onfail="ONOS" + controllerStr +
1825 " hosts don't match Mininet" )
1826
1827 devicesResults = devicesResults and currentDevicesResult
1828 portsResults = portsResults and currentPortsResult
1829 linksResults = linksResults and currentLinksResult
1830 hostsResults = hostsResults and currentHostsResult
Jon Hall94fd0472014-12-08 11:52:42 -08001831
Jon Hall529a37f2015-01-28 10:02:00 -08001832 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001833
Jon Hall6aec96b2015-01-19 14:49:31 -08001834 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -08001835 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001836 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001837 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001838 if "Error" not in hosts[ controller ]:
1839 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001840 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001841 else: # hosts not consistent
Jon Hall8f89dda2015-01-22 16:03:33 -08001842 main.log.report( "hosts from ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001843 " is inconsistent with ONOS1" )
1844 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001845 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001846
1847 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001848 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001849 controllerStr )
1850 consistentHostsResult = main.FALSE
1851 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001852 " hosts response: " +
1853 repr( hosts[ controller ] ) )
1854 utilities.assert_equals(
1855 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001856 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001857 onpass="Hosts view is consistent across all ONOS nodes",
1858 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001859
Jon Hall6aec96b2015-01-19 14:49:31 -08001860 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001861 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001862 for controller in range( len( clusters ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001863 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001864 if "Error" not in clusters[ controller ]:
1865 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001866 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001867 else: # clusters not consistent
1868 main.log.report( "clusters from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001869 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001870 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001871 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001872
1873 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001874 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001875 "from ONOS" + controllerStr )
1876 consistentClustersResult = main.FALSE
1877 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001878 " clusters response: " +
1879 repr( clusters[ controller ] ) )
1880 utilities.assert_equals(
1881 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001882 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001883 onpass="Clusters view is consistent across all ONOS nodes",
1884 onfail="ONOS nodes have different views of clusters" )
1885 # there should always only be one cluster
Jon Hall8f89dda2015-01-22 16:03:33 -08001886 numClusters = len( json.loads( clusters[ 0 ] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001887 clusterResults = main.FALSE
1888 if numClusters == 1:
1889 clusterResults = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001890 utilities.assert_equals(
1891 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001892 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001893 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08001894 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001895
Jon Hall8f89dda2015-01-22 16:03:33 -08001896 topoResult = ( devicesResults and portsResults and linksResults
Jon Hall58c76b72015-02-23 11:09:24 -08001897 and hostsResults and consistentHostsResult
1898 and consistentClustersResult and clusterResults
1899 and ipResult )
Jon Hall94fd0472014-12-08 11:52:42 -08001900
Jon Hall8f89dda2015-01-22 16:03:33 -08001901 topoResult = topoResult and int( count <= 2 )
1902 note = "note it takes about " + str( int( cliTime ) ) + \
1903 " seconds for the test to make all the cli calls to fetch " +\
1904 "the topology from each ONOS instance"
Jon Hall1b8f54a2015-02-04 13:24:20 -08001905 main.log.info(
Jon Hall8f89dda2015-01-22 16:03:33 -08001906 "Very crass estimate for topology discovery/convergence( " +
1907 str( note ) + " ): " + str( elapsed ) + " seconds, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001908 str( count ) + " tries" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001909 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08001910 onpass="Topology Check Test successful",
1911 onfail="Topology Check Test NOT successful" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001912 if topoResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001913 main.log.report( "ONOS topology view matches Mininet topology" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001914
Jon Hall6aec96b2015-01-19 14:49:31 -08001915 def CASE9( self, main ):
1916 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001917 Link s3-s28 down
Jon Hall6aec96b2015-01-19 14:49:31 -08001918 """
1919 import time
1920 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001921
Jon Hall8f89dda2015-01-22 16:03:33 -08001922 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001923
Jon Hall6aec96b2015-01-19 14:49:31 -08001924 description = "Turn off a link to ensure that Link Discovery " +\
Jon Hall58c76b72015-02-23 11:09:24 -08001925 "is working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001926 main.log.report( description )
1927 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001928
Jon Hall6aec96b2015-01-19 14:49:31 -08001929 main.step( "Kill Link between s3 and s28" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001930 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
Jon Hall58c76b72015-02-23 11:09:24 -08001931 main.log.info( "Waiting " + str( linkSleep ) +
1932 " seconds for link down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001933 time.sleep( linkSleep )
1934 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
Jon Hall58c76b72015-02-23 11:09:24 -08001935 onpass="Link down succesful",
1936 onfail="Failed to bring link down" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001937 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -08001938
Jon Hall6aec96b2015-01-19 14:49:31 -08001939 def CASE10( self, main ):
1940 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001941 Link s3-s28 up
Jon Hall6aec96b2015-01-19 14:49:31 -08001942 """
1943 import time
1944 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001945
Jon Hall8f89dda2015-01-22 16:03:33 -08001946 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001947
Jon Hall6aec96b2015-01-19 14:49:31 -08001948 description = "Restore a link to ensure that Link Discovery is " + \
Jon Hall63604932015-02-26 17:09:50 -08001949 "working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001950 main.log.report( description )
1951 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001952
Jon Hall6aec96b2015-01-19 14:49:31 -08001953 main.step( "Bring link between s3 and s28 back up" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001954 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
Jon Hall58c76b72015-02-23 11:09:24 -08001955 main.log.info( "Waiting " + str( linkSleep ) +
1956 " seconds for link up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001957 time.sleep( linkSleep )
1958 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
Jon Hall58c76b72015-02-23 11:09:24 -08001959 onpass="Link up succesful",
1960 onfail="Failed to bring link up" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001961 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -08001962
Jon Hall6aec96b2015-01-19 14:49:31 -08001963 def CASE11( self, main ):
1964 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001965 Switch Down
Jon Hall6aec96b2015-01-19 14:49:31 -08001966 """
1967 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001968 import time
1969
Jon Hall8f89dda2015-01-22 16:03:33 -08001970 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001971
1972 description = "Killing a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001973 main.log.report( description )
1974 main.case( description )
1975 switch = main.params[ 'kill' ][ 'switch' ]
1976 switchDPID = main.params[ 'kill' ][ 'dpid' ]
Jon Hall73cf9cc2014-11-20 22:28:38 -08001977
Jon Hall6aec96b2015-01-19 14:49:31 -08001978 # TODO: Make this switch parameterizable
1979 main.step( "Kill " + switch )
1980 main.log.report( "Deleting " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08001981 main.Mininet1.delSwitch( switch )
1982 main.log.info( "Waiting " + str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001983 " seconds for switch down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001984 time.sleep( switchSleep )
1985 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001986 # Peek at the deleted switch
1987 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001988 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001989 if device and device[ 'available' ] is False:
Jon Hall94fd0472014-12-08 11:52:42 -08001990 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001991 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08001992 onpass="Kill switch succesful",
1993 onfail="Failed to kill switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001994
Jon Hall6aec96b2015-01-19 14:49:31 -08001995 def CASE12( self, main ):
1996 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001997 Switch Up
Jon Hall6aec96b2015-01-19 14:49:31 -08001998 """
1999 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08002000 import time
Jon Hall669173b2014-12-17 11:36:30 -08002001
Jon Hall8f89dda2015-01-22 16:03:33 -08002002 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall6aec96b2015-01-19 14:49:31 -08002003 switch = main.params[ 'kill' ][ 'switch' ]
2004 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2005 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hall73cf9cc2014-11-20 22:28:38 -08002006 description = "Adding a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002007 main.log.report( description )
2008 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002009
Jon Hall6aec96b2015-01-19 14:49:31 -08002010 main.step( "Add back " + switch )
2011 main.log.report( "Adding back " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08002012 main.Mininet1.addSwitch( switch, dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002013 # TODO: New dpid or same? Ask Thomas?
2014 for peer in links:
Jon Hall8f89dda2015-01-22 16:03:33 -08002015 main.Mininet1.addLink( switch, peer )
Jon Hall58c76b72015-02-23 11:09:24 -08002016 main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
2017 count=numControllers,
2018 ip1=ONOS1Ip,
2019 port1=ONOS1Port,
2020 ip2=ONOS2Ip,
2021 port2=ONOS2Port,
2022 ip3=ONOS3Ip,
2023 port3=ONOS3Port,
2024 ip4=ONOS4Ip,
2025 port4=ONOS4Port,
2026 ip5=ONOS5Ip,
2027 port5=ONOS5Port,
2028 ip6=ONOS6Ip,
2029 port6=ONOS6Port,
2030 ip7=ONOS7Ip,
2031 port7=ONOS7Port )
2032 main.log.info( "Waiting " + str( switchSleep ) +
2033 " seconds for switch up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002034 time.sleep( switchSleep )
2035 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002036 # Peek at the deleted switch
2037 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08002038 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002039 if device and device[ 'available' ]:
Jon Hall94fd0472014-12-08 11:52:42 -08002040 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002041 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08002042 onpass="add switch succesful",
2043 onfail="Failed to add switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002044
Jon Hall6aec96b2015-01-19 14:49:31 -08002045 def CASE13( self, main ):
2046 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002047 Clean up
Jon Hall6aec96b2015-01-19 14:49:31 -08002048 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002049 import os
2050 import time
Jon Hall6aec96b2015-01-19 14:49:31 -08002051 # TODO: make use of this elsewhere
2052 ips = []
Jon Hall8f89dda2015-01-22 16:03:33 -08002053 ips.append( ONOS1Ip )
2054 ips.append( ONOS2Ip )
2055 ips.append( ONOS3Ip )
2056 ips.append( ONOS4Ip )
2057 ips.append( ONOS5Ip )
2058 ips.append( ONOS6Ip )
2059 ips.append( ONOS7Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -08002060
2061 # printing colors to terminal
Jon Hall669173b2014-12-17 11:36:30 -08002062 colors = {}
Jon Hall6aec96b2015-01-19 14:49:31 -08002063 colors[ 'cyan' ] = '\033[96m'
2064 colors[ 'purple' ] = '\033[95m'
2065 colors[ 'blue' ] = '\033[94m'
2066 colors[ 'green' ] = '\033[92m'
2067 colors[ 'yellow' ] = '\033[93m'
2068 colors[ 'red' ] = '\033[91m'
2069 colors[ 'end' ] = '\033[0m'
Jon Hall73cf9cc2014-11-20 22:28:38 -08002070 description = "Test Cleanup"
Jon Hall6aec96b2015-01-19 14:49:31 -08002071 main.log.report( description )
2072 main.case( description )
2073 main.step( "Killing tcpdumps" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002074 main.Mininet2.stopTcpdump()
Jon Hall73cf9cc2014-11-20 22:28:38 -08002075
Jon Hall6aec96b2015-01-19 14:49:31 -08002076 main.step( "Checking ONOS Logs for errors" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002077 for i in range( 7 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002078 print colors[ 'purple' ] + "Checking logs for errors on " + \
Jon Hall8f89dda2015-01-22 16:03:33 -08002079 "ONOS" + str( i + 1 ) + ":" + colors[ 'end' ]
2080 print main.ONOSbench.checkLogs( ips[ i ] )
Jon Hall94fd0472014-12-08 11:52:42 -08002081
Jon Hall6aec96b2015-01-19 14:49:31 -08002082 main.step( "Copying MN pcap and ONOS log files to test station" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002083 testname = main.TEST
Jon Hall8f89dda2015-01-22 16:03:33 -08002084 teststationUser = main.params[ 'TESTONUSER' ]
2085 teststationIP = main.params[ 'TESTONIP' ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002086 # NOTE: MN Pcap file is being saved to ~/packet_captures
Jon Hall73cf9cc2014-11-20 22:28:38 -08002087 # scp this file as MN and TestON aren't necessarily the same vm
Jon Hall6aec96b2015-01-19 14:49:31 -08002088 # FIXME: scp
2089 # mn files
2090 # TODO: Load these from params
2091 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002092 logFolder = "/opt/onos/log/"
2093 logFiles = [ "karaf.log", "karaf.log.1" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002094 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002095 dstDir = "~/packet_captures/"
2096 for f in logFiles:
2097 for i in range( 7 ):
2098 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
2099 logFolder + f + " " +
2100 teststationUser + "@" +
2101 teststationIP + ":" +
2102 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002103 "-ONOS" + str( i + 1 ) + "-" +
2104 f )
Jon Hall58c76b72015-02-23 11:09:24 -08002105 main.ONOSbench.handle.expect( "\$" )
2106
Jon Hall6aec96b2015-01-19 14:49:31 -08002107 # std*.log's
2108 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002109 logFolder = "/opt/onos/var/"
2110 logFiles = [ "stderr.log", "stdout.log" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002111 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002112 dstDir = "~/packet_captures/"
2113 for f in logFiles:
2114 for i in range( 7 ):
2115 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
2116 logFolder + f + " " +
2117 teststationUser + "@" +
2118 teststationIP + ":" +
2119 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002120 "-ONOS" + str( i + 1 ) + "-" +
2121 f )
Jon Hall58c76b72015-02-23 11:09:24 -08002122 main.ONOSbench.handle.expect( "\$" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002123 # sleep so scp can finish
2124 time.sleep( 10 )
Jon Hall58c76b72015-02-23 11:09:24 -08002125 main.Mininet1.stopNet()
Jon Hall6aec96b2015-01-19 14:49:31 -08002126 main.step( "Packing and rotating pcap archives" )
2127 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002128
Jon Hall6aec96b2015-01-19 14:49:31 -08002129 # TODO: actually check something here
2130 utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08002131 onpass="Test cleanup successful",
2132 onfail="Test cleanup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002133
Jon Hall6aec96b2015-01-19 14:49:31 -08002134 def CASE14( self, main ):
2135 """
Jon Hall669173b2014-12-17 11:36:30 -08002136 start election app on all onos nodes
Jon Hall6aec96b2015-01-19 14:49:31 -08002137 """
Jon Hall8f89dda2015-01-22 16:03:33 -08002138 leaderResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002139 # install app on onos 1
2140 main.log.info( "Install leadership election app" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002141 main.ONOScli1.featureInstall( "onos-app-election" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002142 # wait for election
2143 # check for leader
Jon Hall8f89dda2015-01-22 16:03:33 -08002144 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002145 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08002146 if leader == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08002147 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08002148 pass
Jon Hall6aec96b2015-01-19 14:49:31 -08002149 elif leader is None:
2150 # No leader elected
2151 main.log.report( "No leader was elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002152 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002153 elif leader == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002154 # error in response
2155 # TODO: add check for "Command not found:" in the driver, this
2156 # means the app isn't loaded
Jon Hall8f89dda2015-01-22 16:03:33 -08002157 main.log.report( "Something is wrong with electionTestLeader" +
Jon Hall6aec96b2015-01-19 14:49:31 -08002158 " function, check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002159 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002160 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002161 # error in response
2162 main.log.report(
Jon Hall8f89dda2015-01-22 16:03:33 -08002163 "Unexpected response from electionTestLeader function:'" +
2164 str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002165 "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002166 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002167
Jon Hall6aec96b2015-01-19 14:49:31 -08002168 # install on other nodes and check for leader.
2169 # Should be onos1 and each app should show the same leader
Jon Hall8f89dda2015-01-22 16:03:33 -08002170 for controller in range( 2, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002171 # loop through ONOScli handlers
2172 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002173 node.featureInstall( "onos-app-election" )
2174 leaderN = node.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002175 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08002176 if leaderN == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08002177 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08002178 pass
2179 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002180 # error in response
2181 # TODO: add check for "Command not found:" in the driver, this
2182 # means the app isn't loaded
2183 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002184 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08002185 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002186 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002187 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08002188 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002189 main.log.report( "ONOS" + str( controller ) + " sees " +
2190 str( leaderN ) +
2191 " as the leader of the election app. Leader" +
2192 " should be " +
2193 str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002194 if leaderResult:
Jon Hall6aec96b2015-01-19 14:49:31 -08002195 main.log.report( "Leadership election tests passed( consistent " +
2196 "view of leader across listeners and a leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002197 "was elected )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002198 utilities.assert_equals(
2199 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002200 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002201 onpass="Leadership election passed",
2202 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002203
Jon Hall6aec96b2015-01-19 14:49:31 -08002204 def CASE15( self, main ):
2205 """
Jon Hall669173b2014-12-17 11:36:30 -08002206 Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -08002207 """
Jon Hall8f89dda2015-01-22 16:03:33 -08002208 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002209 description = "Check that Leadership Election is still functional"
Jon Hall6aec96b2015-01-19 14:49:31 -08002210 main.log.report( description )
2211 main.case( description )
2212 main.step( "Find current leader and withdraw" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002213 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002214 # TODO: do some sanity checking on leader before using it
Jon Hall8f89dda2015-01-22 16:03:33 -08002215 withdrawResult = main.FALSE
2216 if leader == ONOS1Ip:
2217 oldLeader = getattr( main, "ONOScli1" )
2218 elif leader == ONOS2Ip:
2219 oldLeader = getattr( main, "ONOScli2" )
2220 elif leader == ONOS3Ip:
2221 oldLeader = getattr( main, "ONOScli3" )
2222 elif leader == ONOS4Ip:
2223 oldLeader = getattr( main, "ONOScli4" )
2224 elif leader == ONOS5Ip:
2225 oldLeader = getattr( main, "ONOScli5" )
2226 elif leader == ONOS6Ip:
2227 oldLeader = getattr( main, "ONOScli6" )
2228 elif leader == ONOS7Ip:
2229 oldLeader = getattr( main, "ONOScli7" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002230 elif leader is None or leader == main.FALSE:
2231 main.log.report(
2232 "Leader for the election app should be an ONOS node," +
Jon Hall58c76b72015-02-23 11:09:24 -08002233 "instead got '" + str( leader ) + "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002234 leaderResult = main.FALSE
Jon Hall63604932015-02-26 17:09:50 -08002235 oldLeader = None
2236 else:
2237 main.log.error( "Leader election --- why am I HERE?!?")
2238 if oldLeader:
2239 withdrawResult = oldLeader.electionTestWithdraw()
Jon Hall6aec96b2015-01-19 14:49:31 -08002240 utilities.assert_equals(
2241 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002242 actual=withdrawResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002243 onpass="App was withdrawn from election",
2244 onfail="App was not withdrawn from election" )
Jon Hall669173b2014-12-17 11:36:30 -08002245
Jon Hall6aec96b2015-01-19 14:49:31 -08002246 main.step( "Make sure new leader is elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002247 leaderList = []
2248 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002249 # loop through ONOScli handlers
2250 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002251 leaderList.append( node.electionTestLeader() )
2252 for leaderN in leaderList:
Jon Hall669173b2014-12-17 11:36:30 -08002253 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08002254 main.log.report(
Jon Hall58c76b72015-02-23 11:09:24 -08002255 "ONOS" + str( controller ) +
2256 " still sees " + str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002257 " as leader after they withdrew" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002258 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002259 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002260 # error in response
2261 # TODO: add check for "Command not found:" in the driver, this
2262 # means the app isn't loaded
2263 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002264 "electionTestLeader function, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002265 "check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002266 leaderResult = main.FALSE
2267 consistentLeader = main.FALSE
2268 if len( set( leaderList ) ) == 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08002269 main.log.info( "Each Election-app sees '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08002270 str( leaderList[ 0 ] ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002271 "' as the leader" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002272 consistentLeader = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002273 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002274 main.log.report(
2275 "Inconsistent responses for leader of Election-app:" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002276 for n in range( len( leaderList ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002277 main.log.report( "ONOS" + str( n + 1 ) + " response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002278 str( leaderList[ n ] ) )
2279 if leaderResult:
2280 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002281 "view of leader across listeners and a new " +
2282 "leader was elected when the old leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002283 "resigned )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002284 utilities.assert_equals(
2285 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002286 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002287 onpass="Leadership election passed",
2288 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002289
Jon Hall58c76b72015-02-23 11:09:24 -08002290 main.step( "Run for election on old leader( just so everyone " +
2291 "is in the hat )" )
Jon Hall63604932015-02-26 17:09:50 -08002292 if oldLeader:
2293 runResult = oldLeader.electionTestRun()
2294 else:
2295 runResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002296 utilities.assert_equals(
2297 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002298 actual=runResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002299 onpass="App re-ran for election",
2300 onfail="App failed to run for election" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002301 if consistentLeader == main.TRUE:
2302 afterRun = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002303 # verify leader didn't just change
Jon Hall8f89dda2015-01-22 16:03:33 -08002304 if afterRun == leaderList[ 0 ]:
2305 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002306 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08002307 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002308 # TODO: assert on run and withdraw results?
Jon Hall669173b2014-12-17 11:36:30 -08002309
Jon Hall6aec96b2015-01-19 14:49:31 -08002310 utilities.assert_equals(
2311 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002312 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002313 onpass="Leadership election passed",
2314 onfail="Something went wrong with Leadership election after " +
2315 "the old leader re-ran for election" )