blob: 3e7de577b5d2511526f44959ae953b64923689e3 [file] [log] [blame]
Jon Hall6aec96b2015-01-19 14:49:31 -08001"""
Jon Hall73cf9cc2014-11-20 22:28:38 -08002Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign mastership to controllers
8CASE3: Assign intents
9CASE4: Ping across added host intents
10CASE5: Reading state of ONOS
11CASE6: The Failure case.
12CASE7: Check state after control plane failure
13CASE8: Compare topo
14CASE9: Link s3-s28 down
15CASE10: Link s3-s28 up
16CASE11: Switch down
17CASE12: Switch up
18CASE13: Clean up
Jon Hall669173b2014-12-17 11:36:30 -080019CASE14: start election app on all onos nodes
20CASE15: Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -080021"""
Jon Hall8f89dda2015-01-22 16:03:33 -080022
23
Jon Hall73cf9cc2014-11-20 22:28:38 -080024class HATestMinorityRestart:
25
Jon Hall6aec96b2015-01-19 14:49:31 -080026 def __init__( self ):
Jon Hall73cf9cc2014-11-20 22:28:38 -080027 self.default = ''
28
Jon Hall6aec96b2015-01-19 14:49:31 -080029 def CASE1( self, main ):
30 """
Jon Hall73cf9cc2014-11-20 22:28:38 -080031 CASE1 is to compile ONOS and push it to the test machines
32
33 Startup sequence:
Jon Hall73cf9cc2014-11-20 22:28:38 -080034 cell <name>
35 onos-verify-cell
36 NOTE: temporary - onos-remove-raft-logs
Jon Hall58c76b72015-02-23 11:09:24 -080037 onos-uninstall
38 start mininet
39 git pull
40 mvn clean install
41 onos-package
Jon Hall73cf9cc2014-11-20 22:28:38 -080042 onos-install -f
43 onos-wait-for-start
Jon Hall58c76b72015-02-23 11:09:24 -080044 start cli sessions
45 start tcpdump
Jon Hall6aec96b2015-01-19 14:49:31 -080046 """
47 main.log.report(
48 "ONOS HA test: Restart minority of ONOS nodes - initialization" )
49 main.case( "Setting up test environment" )
50 # TODO: save all the timers and output them for plotting
Jon Hall73cf9cc2014-11-20 22:28:38 -080051
52 # load some vairables from the params file
Jon Hall8f89dda2015-01-22 16:03:33 -080053 PULLCODE = False
Jon Hall6aec96b2015-01-19 14:49:31 -080054 if main.params[ 'Git' ] == 'True':
Jon Hall8f89dda2015-01-22 16:03:33 -080055 PULLCODE = True
Jon Hall529a37f2015-01-28 10:02:00 -080056 gitBranch = main.params[ 'branch' ]
Jon Hall8f89dda2015-01-22 16:03:33 -080057 cellName = main.params[ 'ENV' ][ 'cellName' ]
Jon Hall6aec96b2015-01-19 14:49:31 -080058
59 # set global variables
Jon Hall8f89dda2015-01-22 16:03:33 -080060 global ONOS1Ip
61 global ONOS1Port
62 global ONOS2Ip
63 global ONOS2Port
64 global ONOS3Ip
65 global ONOS3Port
66 global ONOS4Ip
67 global ONOS4Port
68 global ONOS5Ip
69 global ONOS5Port
70 global ONOS6Ip
71 global ONOS6Port
72 global ONOS7Ip
73 global ONOS7Port
74 global numControllers
Jon Hall73cf9cc2014-11-20 22:28:38 -080075
Jon Hall8f89dda2015-01-22 16:03:33 -080076 ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
77 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
78 ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
79 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
80 ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
81 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
82 ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
83 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
84 ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
85 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
86 ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
87 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
88 ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
89 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
90 numControllers = int( main.params[ 'num_controllers' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -080091
Jon Hall6aec96b2015-01-19 14:49:31 -080092 main.step( "Applying cell variable to environment" )
Jon Hall8f89dda2015-01-22 16:03:33 -080093 cellResult = main.ONOSbench.setCell( cellName )
94 verifyResult = main.ONOSbench.verifyCell()
Jon Hall73cf9cc2014-11-20 22:28:38 -080095
Jon Hall6aec96b2015-01-19 14:49:31 -080096 # FIXME:this is short term fix
97 main.log.report( "Removing raft logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -080098 main.ONOSbench.onosRemoveRaftLogs()
Jon Hall6aec96b2015-01-19 14:49:31 -080099 main.log.report( "Uninstalling ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800100 main.ONOSbench.onosUninstall( ONOS1Ip )
101 main.ONOSbench.onosUninstall( ONOS2Ip )
102 main.ONOSbench.onosUninstall( ONOS3Ip )
103 main.ONOSbench.onosUninstall( ONOS4Ip )
104 main.ONOSbench.onosUninstall( ONOS5Ip )
105 main.ONOSbench.onosUninstall( ONOS6Ip )
106 main.ONOSbench.onosUninstall( ONOS7Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800107
Jon Hall8f89dda2015-01-22 16:03:33 -0800108 cleanInstallResult = main.TRUE
109 gitPullResult = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800110
Jon Hall97f31752015-02-04 12:01:04 -0800111 main.step( "Starting Mininet" )
112 main.Mininet1.startNet( )
113
Jon Hall6aec96b2015-01-19 14:49:31 -0800114 main.step( "Compiling the latest version of ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800115 if PULLCODE:
Jon Hall58c76b72015-02-23 11:09:24 -0800116 main.step( "Git checkout and pull " + gitBranch )
Jon Hall529a37f2015-01-28 10:02:00 -0800117 main.ONOSbench.gitCheckout( gitBranch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800118 gitPullResult = main.ONOSbench.gitPull()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800119
Jon Hall6aec96b2015-01-19 14:49:31 -0800120 main.step( "Using mvn clean & install" )
Jon Hall529a37f2015-01-28 10:02:00 -0800121 cleanInstallResult = main.ONOSbench.cleanInstall()
122 else:
123 main.log.warn( "Did not pull new code so skipping mvn " +
124 "clean install" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800125 main.ONOSbench.getVersion( report=True )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800126
Jon Hall6aec96b2015-01-19 14:49:31 -0800127 main.step( "Creating ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800128 packageResult = main.ONOSbench.onosPackage()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800129
Jon Hall6aec96b2015-01-19 14:49:31 -0800130 main.step( "Installing ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800131 onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
132 node=ONOS1Ip )
133 onos2InstallResult = main.ONOSbench.onosInstall( options="-f",
134 node=ONOS2Ip )
135 onos3InstallResult = main.ONOSbench.onosInstall( options="-f",
136 node=ONOS3Ip )
137 onos4InstallResult = main.ONOSbench.onosInstall( options="-f",
138 node=ONOS4Ip )
139 onos5InstallResult = main.ONOSbench.onosInstall( options="-f",
140 node=ONOS5Ip )
141 onos6InstallResult = main.ONOSbench.onosInstall( options="-f",
142 node=ONOS6Ip )
143 onos7InstallResult = main.ONOSbench.onosInstall( options="-f",
144 node=ONOS7Ip )
145 onosInstallResult = onos1InstallResult and onos2InstallResult\
146 and onos3InstallResult and onos4InstallResult\
147 and onos5InstallResult and onos6InstallResult\
148 and onos7InstallResult
Jon Hall73cf9cc2014-11-20 22:28:38 -0800149
Jon Hall6aec96b2015-01-19 14:49:31 -0800150 main.step( "Checking if ONOS is up yet" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800151 for i in range( 2 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800152 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
153 if not onos1Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800154 main.log.report( "ONOS1 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800155 main.ONOSbench.onosStop( ONOS1Ip )
156 main.ONOSbench.onosStart( ONOS1Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800157 onos2Isup = main.ONOSbench.isup( ONOS2Ip )
158 if not onos2Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800159 main.log.report( "ONOS2 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800160 main.ONOSbench.onosStop( ONOS2Ip )
161 main.ONOSbench.onosStart( ONOS2Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800162 onos3Isup = main.ONOSbench.isup( ONOS3Ip )
163 if not onos3Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800164 main.log.report( "ONOS3 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800165 main.ONOSbench.onosStop( ONOS3Ip )
166 main.ONOSbench.onosStart( ONOS3Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800167 onos4Isup = main.ONOSbench.isup( ONOS4Ip )
168 if not onos4Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800169 main.log.report( "ONOS4 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800170 main.ONOSbench.onosStop( ONOS4Ip )
171 main.ONOSbench.onosStart( ONOS4Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800172 onos5Isup = main.ONOSbench.isup( ONOS5Ip )
173 if not onos5Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800174 main.log.report( "ONOS5 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800175 main.ONOSbench.onosStop( ONOS5Ip )
176 main.ONOSbench.onosStart( ONOS5Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800177 onos6Isup = main.ONOSbench.isup( ONOS6Ip )
178 if not onos6Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800179 main.log.report( "ONOS6 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800180 main.ONOSbench.onosStop( ONOS6Ip )
181 main.ONOSbench.onosStart( ONOS6Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800182 onos7Isup = main.ONOSbench.isup( ONOS7Ip )
183 if not onos7Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800184 main.log.report( "ONOS7 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800185 main.ONOSbench.onosStop( ONOS7Ip )
186 main.ONOSbench.onosStart( ONOS7Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800187 onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
188 and onos4Isup and onos5Isup and onos6Isup and onos7Isup
189 if onosIsupResult == main.TRUE:
Jon Hall94fd0472014-12-08 11:52:42 -0800190 break
Jon Hall73cf9cc2014-11-20 22:28:38 -0800191
Jon Hall8f89dda2015-01-22 16:03:33 -0800192 cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
193 cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
194 cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
195 cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
196 cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
197 cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
198 cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
199 cliResults = cliResult1 and cliResult2 and cliResult3 and\
200 cliResult4 and cliResult5 and cliResult6 and cliResult7
Jon Hall73cf9cc2014-11-20 22:28:38 -0800201
Jon Hall6aec96b2015-01-19 14:49:31 -0800202 main.step( "Start Packet Capture MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800203 main.Mininet2.startTcpdump(
Jon Hall6aec96b2015-01-19 14:49:31 -0800204 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
205 + "-MN.pcap",
206 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
207 port=main.params[ 'MNtcpdump' ][ 'port' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800208
Jon Hall8f89dda2015-01-22 16:03:33 -0800209 case1Result = ( cleanInstallResult and packageResult and
210 cellResult and verifyResult and onosInstallResult
211 and onosIsupResult and cliResults )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800212
Jon Hall8f89dda2015-01-22 16:03:33 -0800213 utilities.assert_equals( expect=main.TRUE, actual=case1Result,
Jon Hall58c76b72015-02-23 11:09:24 -0800214 onpass="Test startup successful",
215 onfail="Test startup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800216
Jon Hall8f89dda2015-01-22 16:03:33 -0800217 if case1Result == main.FALSE:
Jon Hall94fd0472014-12-08 11:52:42 -0800218 main.cleanup()
219 main.exit()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800220
Jon Hall6aec96b2015-01-19 14:49:31 -0800221 def CASE2( self, main ):
222 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800223 Assign mastership to controllers
Jon Hall6aec96b2015-01-19 14:49:31 -0800224 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800225 import re
226
Jon Hall6aec96b2015-01-19 14:49:31 -0800227 main.log.report( "Assigning switches to controllers" )
228 main.case( "Assigning Controllers" )
229 main.step( "Assign switches to controllers" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800230
Jon Hall6aec96b2015-01-19 14:49:31 -0800231 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800232 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -0800233 sw=str( i ),
Jon Hall8f89dda2015-01-22 16:03:33 -0800234 count=numControllers,
235 ip1=ONOS1Ip, port1=ONOS1Port,
236 ip2=ONOS2Ip, port2=ONOS2Port,
237 ip3=ONOS3Ip, port3=ONOS3Port,
238 ip4=ONOS4Ip, port4=ONOS4Port,
239 ip5=ONOS5Ip, port5=ONOS5Port,
240 ip6=ONOS6Ip, port6=ONOS6Port,
241 ip7=ONOS7Ip, port7=ONOS7Port )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800242
Jon Hall8f89dda2015-01-22 16:03:33 -0800243 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800244 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800245 response = main.Mininet1.getSwController( "s" + str( i ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800246 try:
Jon Hall6aec96b2015-01-19 14:49:31 -0800247 main.log.info( str( response ) )
Jon Hallfebb1c72015-03-05 13:30:09 -0800248 except Exception:
Jon Hall6aec96b2015-01-19 14:49:31 -0800249 main.log.info( repr( response ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800250 if re.search( "tcp:" + ONOS1Ip, response )\
251 and re.search( "tcp:" + ONOS2Ip, response )\
252 and re.search( "tcp:" + ONOS3Ip, response )\
253 and re.search( "tcp:" + ONOS4Ip, response )\
254 and re.search( "tcp:" + ONOS5Ip, response )\
255 and re.search( "tcp:" + ONOS6Ip, response )\
256 and re.search( "tcp:" + ONOS7Ip, response ):
257 mastershipCheck = mastershipCheck and main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800258 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800259 mastershipCheck = main.FALSE
260 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800261 main.log.report( "Switch mastership assigned correctly" )
262 utilities.assert_equals(
263 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800264 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800265 onpass="Switch mastership assigned correctly",
266 onfail="Switches not assigned correctly to controllers" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800267
Jon Hall6aec96b2015-01-19 14:49:31 -0800268 # Manually assign mastership to the controller we want
Jon Hall8f89dda2015-01-22 16:03:33 -0800269 roleCall = main.TRUE
270 roleCheck = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -0800271 try:
272 # Assign switch
273 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
274 assert deviceId, "No device id for s1 in ONOS"
Jon Hall8f89dda2015-01-22 16:03:33 -0800275 roleCall = roleCall and main.ONOScli1.deviceRole(
276 deviceId,
Jon Hall58c76b72015-02-23 11:09:24 -0800277 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800278 # Check assignment
Jon Hall58c76b72015-02-23 11:09:24 -0800279 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800280 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800281 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800282 roleCheck = roleCheck and main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800283
Jon Hall58c76b72015-02-23 11:09:24 -0800284 # Assign switch
285 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
286 assert deviceId, "No device id for s28 in ONOS"
Jon Hall8f89dda2015-01-22 16:03:33 -0800287 roleCall = roleCall and main.ONOScli1.deviceRole(
288 deviceId,
Jon Hall58c76b72015-02-23 11:09:24 -0800289 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800290 # Check assignment
Jon Hall58c76b72015-02-23 11:09:24 -0800291 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800292 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800293 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800294 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800295
Jon Hall58c76b72015-02-23 11:09:24 -0800296 # Assign switch
297 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
298 assert deviceId, "No device id for s2 in ONOS"
299 roleCall = roleCall and main.ONOScli1.deviceRole(
300 deviceId,
301 ONOS2Ip )
302 # Check assignment
303 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
304 roleCheck = roleCheck and main.TRUE
305 else:
306 roleCheck = roleCheck and main.FALSE
307
308 # Assign switch
309 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
310 assert deviceId, "No device id for s3 in ONOS"
311 roleCall = roleCall and main.ONOScli1.deviceRole(
312 deviceId,
313 ONOS2Ip )
314 # Check assignment
315 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
316 roleCheck = roleCheck and main.TRUE
317 else:
318 roleCheck = roleCheck and main.FALSE
319
320 # Assign switch
321 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
322 assert deviceId, "No device id for s5 in ONOS"
323 roleCall = roleCall and main.ONOScli1.deviceRole(
324 deviceId,
325 ONOS3Ip )
326 # Check assignment
327 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
328 roleCheck = roleCheck and main.TRUE
329 else:
330 roleCheck = roleCheck and main.FALSE
331
332 # Assign switch
333 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
334 assert deviceId, "No device id for s6 in ONOS"
335 roleCall = roleCall and main.ONOScli1.deviceRole(
336 deviceId,
337 ONOS3Ip )
338 # Check assignment
339 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
340 roleCheck = roleCheck and main.TRUE
341 else:
342 roleCheck = roleCheck and main.FALSE
343
344 # Assign switch
345 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
346 assert deviceId, "No device id for s4 in ONOS"
347 roleCall = roleCall and main.ONOScli1.deviceRole(
348 deviceId,
349 ONOS4Ip )
350 # Check assignment
351 if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
352 roleCheck = roleCheck and main.TRUE
353 else:
354 roleCheck = roleCheck and main.FALSE
355
356 for i in range( 8, 18 ):
357 dpid = '3' + str( i ).zfill( 3 )
358 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
359 assert deviceId, "No device id for s%i in ONOS" % i
360 roleCall = roleCall and main.ONOScli1.deviceRole(
361 deviceId,
362 ONOS5Ip )
363 # Check assignment
364 if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
365 roleCheck = roleCheck and main.TRUE
366 else:
367 roleCheck = roleCheck and main.FALSE
368
369 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
370 assert deviceId, "No device id for s7 in ONOS"
371 roleCall = roleCall and main.ONOScli1.deviceRole(
372 deviceId,
373 ONOS6Ip )
374 # Check assignment
375 if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
376 roleCheck = roleCheck and main.TRUE
377 else:
378 roleCheck = roleCheck and main.FALSE
379
380 for i in range( 18, 28 ):
381 dpid = '6' + str( i ).zfill( 3 )
382 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
383 assert deviceId, "No device id for s%i in ONOS" % i
384 roleCall = roleCall and main.ONOScli1.deviceRole(
385 deviceId,
386 ONOS7Ip )
387 # Check assignment
388 if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
389 roleCheck = roleCheck and main.TRUE
390 else:
391 roleCheck = roleCheck and main.FALSE
392 except ( AttributeError, AssertionError ):
393 main.log.exception( "Something is wrong with ONOS device view" )
394 main.log.info( main.ONOScli1.devices() )
395
Jon Hall6aec96b2015-01-19 14:49:31 -0800396 utilities.assert_equals(
397 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800398 actual=roleCall,
Jon Hall6aec96b2015-01-19 14:49:31 -0800399 onpass="Re-assigned switch mastership to designated controller",
Jon Hall8f89dda2015-01-22 16:03:33 -0800400 onfail="Something wrong with deviceRole calls" )
Jon Hall94fd0472014-12-08 11:52:42 -0800401
Jon Hall6aec96b2015-01-19 14:49:31 -0800402 utilities.assert_equals(
403 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800404 actual=roleCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800405 onpass="Switches were successfully reassigned to designated " +
406 "controller",
407 onfail="Switches were not successfully reassigned" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800408 mastershipCheck = mastershipCheck and roleCall and roleCheck
409 utilities.assert_equals( expect=main.TRUE, actual=mastershipCheck,
Jon Hall21270ac2015-02-16 17:59:55 -0800410 onpass="Switch mastership correctly assigned",
411 onfail="Error in (re)assigning switch" +
412 " mastership" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800413
Jon Hall6aec96b2015-01-19 14:49:31 -0800414 def CASE3( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800415 """
416 Assign intents
Jon Hall73cf9cc2014-11-20 22:28:38 -0800417 """
418 import time
Jon Hallfebb1c72015-03-05 13:30:09 -0800419 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800420 main.log.report( "Adding host intents" )
421 main.case( "Adding host Intents" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800422
Jon Hall8f89dda2015-01-22 16:03:33 -0800423 main.step( "Discovering Hosts( Via pingall for now )" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800424 # FIXME: Once we have a host discovery mechanism, use that instead
Jon Hall73cf9cc2014-11-20 22:28:38 -0800425
Jon Hall6aec96b2015-01-19 14:49:31 -0800426 # install onos-app-fwd
427 main.log.info( "Install reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800428 main.ONOScli1.featureInstall( "onos-app-fwd" )
429 main.ONOScli2.featureInstall( "onos-app-fwd" )
430 main.ONOScli3.featureInstall( "onos-app-fwd" )
431 main.ONOScli4.featureInstall( "onos-app-fwd" )
432 main.ONOScli5.featureInstall( "onos-app-fwd" )
433 main.ONOScli6.featureInstall( "onos-app-fwd" )
434 main.ONOScli7.featureInstall( "onos-app-fwd" )
Jon Hall94fd0472014-12-08 11:52:42 -0800435
Jon Hall6aec96b2015-01-19 14:49:31 -0800436 # REACTIVE FWD test
Jon Hall8f89dda2015-01-22 16:03:33 -0800437 pingResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800438 time1 = time.time()
Jon Hall8f89dda2015-01-22 16:03:33 -0800439 pingResult = main.Mininet1.pingall()
Jon Hall529a37f2015-01-28 10:02:00 -0800440 utilities.assert_equals(
441 expect=main.TRUE,
442 actual=pingResult,
443 onpass="Reactive Pingall test passed",
444 onfail="Reactive Pingall failed, one or more ping pairs failed" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800445 time2 = time.time()
Jon Hall6aec96b2015-01-19 14:49:31 -0800446 main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800447
Jon Hall6aec96b2015-01-19 14:49:31 -0800448 # uninstall onos-app-fwd
449 main.log.info( "Uninstall reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800450 main.ONOScli1.featureUninstall( "onos-app-fwd" )
451 main.ONOScli2.featureUninstall( "onos-app-fwd" )
452 main.ONOScli3.featureUninstall( "onos-app-fwd" )
453 main.ONOScli4.featureUninstall( "onos-app-fwd" )
454 main.ONOScli5.featureUninstall( "onos-app-fwd" )
455 main.ONOScli6.featureUninstall( "onos-app-fwd" )
456 main.ONOScli7.featureUninstall( "onos-app-fwd" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800457 # timeout for fwd flows
458 time.sleep( 10 )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800459
Jon Hall6aec96b2015-01-19 14:49:31 -0800460 main.step( "Add host intents" )
Jon Hall58c76b72015-02-23 11:09:24 -0800461 intentIds = []
Jon Hall6aec96b2015-01-19 14:49:31 -0800462 # TODO: move the host numbers to params
Jon Hall58c76b72015-02-23 11:09:24 -0800463 # Maybe look at all the paths we ping?
Jon Hall8f89dda2015-01-22 16:03:33 -0800464 intentAddResult = True
Jon Hall58c76b72015-02-23 11:09:24 -0800465 hostResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800466 for i in range( 8, 18 ):
467 main.log.info( "Adding host intent between h" + str( i ) +
468 " and h" + str( i + 10 ) )
469 host1 = "00:00:00:00:00:" + \
470 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
471 host2 = "00:00:00:00:00:" + \
472 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800473 # NOTE: getHost can return None
474 host1Dict = main.ONOScli1.getHost( host1 )
475 host2Dict = main.ONOScli1.getHost( host2 )
476 host1Id = None
477 host2Id = None
478 if host1Dict and host2Dict:
479 host1Id = host1Dict.get( 'id', None )
480 host2Id = host2Dict.get( 'id', None )
Jon Hall8f89dda2015-01-22 16:03:33 -0800481 if host1Id and host2Id:
Jon Hall58c76b72015-02-23 11:09:24 -0800482 # Changed onos node to test something
483 tmpId = main.ONOScli4.addHostIntent(
Jon Hall8f89dda2015-01-22 16:03:33 -0800484 host1Id,
485 host2Id )
Jon Hall63604932015-02-26 17:09:50 -0800486 if tmpId:
487 main.log.info( "Added intent with id: " + tmpId )
488 intentIds.append( tmpId )
489 else:
490 main.log.error( "addHostIntent reutrned None" )
Jon Hall669173b2014-12-17 11:36:30 -0800491 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800492 main.log.error( "Error, getHost() failed" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800493 main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
494 sort_keys=True,
495 indent=4,
496 separators=( ',', ': ' ) ) )
Jon Hall58c76b72015-02-23 11:09:24 -0800497 hostResult = main.FALSE
498 onosIds = main.ONOScli1.getAllIntentsId()
499 main.log.info( "Submitted intents: " + str( intentIds ) )
500 main.log.info( "Intents in ONOS: " + str( onosIds ) )
501 for intent in intentIds:
502 if intent in onosIds:
503 pass # intent submitted is still in onos
504 else:
505 intentAddResult = False
Jon Hall1b8f54a2015-02-04 13:24:20 -0800506 # Print the intent states
Jon Hall58c76b72015-02-23 11:09:24 -0800507 intents = main.ONOScli1.intents()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800508 intentStates = []
Jon Hall63604932015-02-26 17:09:50 -0800509 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800510 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
511 count = 0
Jon Hall1b8f54a2015-02-04 13:24:20 -0800512 for intent in json.loads( intents ): # Iter through intents of a node
Jon Hall58c76b72015-02-23 11:09:24 -0800513 state = intent.get( 'state', None )
Jon Hall63604932015-02-26 17:09:50 -0800514 if "INSTALLED" not in state:
515 installedCheck = False
Jon Hall58c76b72015-02-23 11:09:24 -0800516 intentId = intent.get( 'id', None )
517 intentStates.append( ( intentId, state ) )
518 # add submitted intents not in the store
519 tmplist = [ i for i, s in intentStates ]
520 missingIntents = False
521 for i in intentIds:
522 if i not in tmplist:
523 intentStates.append( ( i, " - " ) )
524 missingIntents = True
525 intentStates.sort()
526 for i, s in intentStates:
527 count += 1
528 main.log.info( "%-6s%-15s%-15s" %
529 ( str( count ), str( i ), str( s ) ) )
Jon Hall63604932015-02-26 17:09:50 -0800530 main.ONOScli1.leaders()
531 main.ONOScli1.partitions()
532 # for node in nodes:
533 # node.pendingMap()
534 pendingMap = main.ONOScli1.pendingMap()
535 main.ONOScli2.pendingMap()
536 main.ONOScli3.pendingMap()
537 main.ONOScli4.pendingMap()
538 main.ONOScli5.pendingMap()
539 main.ONOScli6.pendingMap()
540 main.ONOScli7.pendingMap()
Jon Hall58c76b72015-02-23 11:09:24 -0800541 intentAddResult = bool( pingResult and hostResult and intentAddResult
Jon Hall63604932015-02-26 17:09:50 -0800542 and not missingIntents and installedCheck )
Jon Hall6aec96b2015-01-19 14:49:31 -0800543 utilities.assert_equals(
544 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800545 actual=intentAddResult,
Jon Hall529a37f2015-01-28 10:02:00 -0800546 onpass="Pushed host intents to ONOS",
547 onfail="Error in pushing host intents to ONOS" )
Jon Hall58c76b72015-02-23 11:09:24 -0800548
Jon Hall63604932015-02-26 17:09:50 -0800549 if not intentAddResult or "key" in pendingMap:
Jon Hall58c76b72015-02-23 11:09:24 -0800550 import time
Jon Hall63604932015-02-26 17:09:50 -0800551 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800552 main.log.info( "Sleeping 60 seconds to see if intents are found" )
553 time.sleep( 60 )
554 onosIds = main.ONOScli1.getAllIntentsId()
555 main.log.info( "Submitted intents: " + str( intentIds ) )
556 main.log.info( "Intents in ONOS: " + str( onosIds ) )
557 # Print the intent states
558 intents = main.ONOScli1.intents()
559 intentStates = []
560 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
561 count = 0
562 for intent in json.loads( intents ):
563 # Iter through intents of a node
564 state = intent.get( 'state', None )
Jon Hall63604932015-02-26 17:09:50 -0800565 if "INSTALLED" not in state:
566 installedCheck = False
Jon Hall58c76b72015-02-23 11:09:24 -0800567 intentId = intent.get( 'id', None )
568 intentStates.append( ( intentId, state ) )
569 # add submitted intents not in the store
570 tmplist = [ i for i, s in intentStates ]
571 for i in intentIds:
572 if i not in tmplist:
573 intentStates.append( ( i, " - " ) )
574 intentStates.sort()
575 for i, s in intentStates:
576 count += 1
577 main.log.info( "%-6s%-15s%-15s" %
578 ( str( count ), str( i ), str( s ) ) )
Jon Hall63604932015-02-26 17:09:50 -0800579 main.ONOScli1.leaders()
580 main.ONOScli1.pendingMap()
581 main.ONOScli2.pendingMap()
582 main.ONOScli3.pendingMap()
583 main.ONOScli4.pendingMap()
584 main.ONOScli5.pendingMap()
585 main.ONOScli6.pendingMap()
586 main.ONOScli7.pendingMap()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800587
Jon Hall6aec96b2015-01-19 14:49:31 -0800588 def CASE4( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800589 """
590 Ping across added host intents
591 """
Jon Hallfebb1c72015-03-05 13:30:09 -0800592 import json
Jon Hall73cf9cc2014-11-20 22:28:38 -0800593 description = " Ping across added host intents"
Jon Hall6aec96b2015-01-19 14:49:31 -0800594 main.log.report( description )
595 main.case( description )
Jon Hall8f89dda2015-01-22 16:03:33 -0800596 PingResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800597 for i in range( 8, 18 ):
Jon Hall21270ac2015-02-16 17:59:55 -0800598 ping = main.Mininet1.pingHost( src="h" + str( i ),
599 target="h" + str( i + 10 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800600 PingResult = PingResult and ping
Jon Hall6aec96b2015-01-19 14:49:31 -0800601 if ping == main.FALSE:
602 main.log.warn( "Ping failed between h" + str( i ) +
603 " and h" + str( i + 10 ) )
604 elif ping == main.TRUE:
605 main.log.info( "Ping test passed!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800606 # Don't set PingResult or you'd override failures
Jon Hall8f89dda2015-01-22 16:03:33 -0800607 if PingResult == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800608 main.log.report(
609 "Intents have not been installed correctly, pings failed." )
Jon Hall58c76b72015-02-23 11:09:24 -0800610 # TODO: pretty print
Jon Hall529a37f2015-01-28 10:02:00 -0800611 main.log.warn( "ONSO1 intents: " )
612 main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
613 sort_keys=True,
614 indent=4,
615 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800616 if PingResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800617 main.log.report(
618 "Intents have been installed correctly and verified by pings" )
619 utilities.assert_equals(
620 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800621 actual=PingResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800622 onpass="Intents have been installed correctly and pings work",
623 onfail="Intents have not been installed correctly, pings failed." )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800624
Jon Hall63604932015-02-26 17:09:50 -0800625 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800626 if PingResult is not main.TRUE:
627 # Print the intent states
628 intents = main.ONOScli1.intents()
629 intentStates = []
630 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
631 count = 0
632 # Iter through intents of a node
633 for intent in json.loads( intents ):
634 state = intent.get( 'state', None )
Jon Hall63604932015-02-26 17:09:50 -0800635 if "INSTALLED" not in state:
636 installedCheck = False
Jon Hall58c76b72015-02-23 11:09:24 -0800637 intentId = intent.get( 'id', None )
638 intentStates.append( ( intentId, state ) )
639 intentStates.sort()
640 for i, s in intentStates:
641 count += 1
642 main.log.info( "%-6s%-15s%-15s" %
643 ( str( count ), str( i ), str( s ) ) )
Jon Hall63604932015-02-26 17:09:50 -0800644 main.ONOScli1.leaders()
645 main.ONOScli1.partitions()
646 main.ONOScli1.pendingMap()
647 main.ONOScli2.pendingMap()
648 main.ONOScli3.pendingMap()
649 main.ONOScli4.pendingMap()
650 main.ONOScli5.pendingMap()
651 main.ONOScli6.pendingMap()
652 main.ONOScli7.pendingMap()
653 if not installedCheck:
654 main.log.info( "Waiting 60 seconds to see if intent states change" )
655 time.sleep( 60 )
656 # Print the intent states
657 intents = main.ONOScli1.intents()
658 intentStates = []
659 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
660 count = 0
661 # Iter through intents of a node
662 for intent in json.loads( intents ):
663 state = intent.get( 'state', None )
664 if "INSTALLED" not in state:
665 installedCheck = False
666 intentId = intent.get( 'id', None )
667 intentStates.append( ( intentId, state ) )
668 intentStates.sort()
669 for i, s in intentStates:
670 count += 1
671 main.log.info( "%-6s%-15s%-15s" %
672 ( str( count ), str( i ), str( s ) ) )
673 main.ONOScli1.leaders()
674 main.ONOScli1.partitions()
675 main.ONOScli1.pendingMap()
676 main.ONOScli2.pendingMap()
677 main.ONOScli3.pendingMap()
678 main.ONOScli4.pendingMap()
679 main.ONOScli5.pendingMap()
680 main.ONOScli6.pendingMap()
681 main.ONOScli7.pendingMap()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800682
Jon Hall6aec96b2015-01-19 14:49:31 -0800683 def CASE5( self, main ):
684 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800685 Reading state of ONOS
Jon Hall6aec96b2015-01-19 14:49:31 -0800686 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800687 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800688 # assumes that sts is already in you PYTHONPATH
689 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -0800690
Jon Hall6aec96b2015-01-19 14:49:31 -0800691 main.log.report( "Setting up and gathering data for current state" )
692 main.case( "Setting up and gathering data for current state" )
693 # The general idea for this test case is to pull the state of
694 # ( intents,flows, topology,... ) from each ONOS node
695 # We can then compare them with eachother and also with past states
Jon Hall73cf9cc2014-11-20 22:28:38 -0800696
Jon Hall6aec96b2015-01-19 14:49:31 -0800697 main.step( "Get the Mastership of each switch from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800698 global mastershipState
699 mastershipState = []
Jon Hall94fd0472014-12-08 11:52:42 -0800700
Jon Hall6aec96b2015-01-19 14:49:31 -0800701 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -0800702 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
703 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
704 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
705 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
706 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
707 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
708 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
709 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
710 ONOS3MasterNotNull and ONOS4MasterNotNull and\
711 ONOS5MasterNotNull and ONOS6MasterNotNull and\
712 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -0800713 utilities.assert_equals(
714 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800715 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -0800716 onpass="Each device has a master",
717 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -0800718
Jon Hall8f89dda2015-01-22 16:03:33 -0800719 ONOS1Mastership = main.ONOScli1.roles()
720 ONOS2Mastership = main.ONOScli2.roles()
721 ONOS3Mastership = main.ONOScli3.roles()
722 ONOS4Mastership = main.ONOScli4.roles()
723 ONOS5Mastership = main.ONOScli5.roles()
724 ONOS6Mastership = main.ONOScli6.roles()
725 ONOS7Mastership = main.ONOScli7.roles()
726 if "Error" in ONOS1Mastership or not ONOS1Mastership\
727 or "Error" in ONOS2Mastership or not ONOS2Mastership\
728 or "Error" in ONOS3Mastership or not ONOS3Mastership\
729 or "Error" in ONOS4Mastership or not ONOS4Mastership\
730 or "Error" in ONOS5Mastership or not ONOS5Mastership\
731 or "Error" in ONOS6Mastership or not ONOS6Mastership\
732 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -0800733 main.log.report( "Error in getting ONOS roles" )
734 main.log.warn(
735 "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800736 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800737 main.log.warn(
738 "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800739 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800740 main.log.warn(
741 "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800742 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800743 main.log.warn(
744 "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800745 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800746 main.log.warn(
747 "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800748 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800749 main.log.warn(
750 "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800751 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800752 main.log.warn(
753 "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800754 repr( ONOS7Mastership ) )
755 consistentMastership = main.FALSE
756 elif ONOS1Mastership == ONOS2Mastership\
757 and ONOS1Mastership == ONOS3Mastership\
758 and ONOS1Mastership == ONOS4Mastership\
759 and ONOS1Mastership == ONOS5Mastership\
760 and ONOS1Mastership == ONOS6Mastership\
761 and ONOS1Mastership == ONOS7Mastership:
762 mastershipState = ONOS1Mastership
763 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800764 main.log.report(
765 "Switch roles are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800766 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800767 main.log.warn(
768 "ONOS1 roles: ",
769 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800770 json.loads( ONOS1Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800771 sort_keys=True,
772 indent=4,
773 separators=(
774 ',',
775 ': ' ) ) )
776 main.log.warn(
777 "ONOS2 roles: ",
778 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800779 json.loads( ONOS2Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800780 sort_keys=True,
781 indent=4,
782 separators=(
783 ',',
784 ': ' ) ) )
785 main.log.warn(
786 "ONOS3 roles: ",
787 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800788 json.loads( ONOS3Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800789 sort_keys=True,
790 indent=4,
791 separators=(
792 ',',
793 ': ' ) ) )
794 main.log.warn(
795 "ONOS4 roles: ",
796 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800797 json.loads( ONOS4Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800798 sort_keys=True,
799 indent=4,
800 separators=(
801 ',',
802 ': ' ) ) )
803 main.log.warn(
804 "ONOS5 roles: ",
805 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800806 json.loads( ONOS5Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800807 sort_keys=True,
808 indent=4,
809 separators=(
810 ',',
811 ': ' ) ) )
812 main.log.warn(
813 "ONOS6 roles: ",
814 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800815 json.loads( ONOS6Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800816 sort_keys=True,
817 indent=4,
818 separators=(
819 ',',
820 ': ' ) ) )
821 main.log.warn(
822 "ONOS7 roles: ",
823 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800824 json.loads( ONOS7Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800825 sort_keys=True,
826 indent=4,
827 separators=(
828 ',',
829 ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800830 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800831 utilities.assert_equals(
832 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800833 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -0800834 onpass="Switch roles are consistent across all ONOS nodes",
835 onfail="ONOS nodes have different views of switch roles" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800836
Jon Hall6aec96b2015-01-19 14:49:31 -0800837 main.step( "Get the intents from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800838 global intentState
839 intentState = []
840 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
841 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
842 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
843 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
844 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
845 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
846 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
847 intentCheck = main.FALSE
848 if "Error" in ONOS1Intents or not ONOS1Intents\
849 or "Error" in ONOS2Intents or not ONOS2Intents\
850 or "Error" in ONOS3Intents or not ONOS3Intents\
851 or "Error" in ONOS4Intents or not ONOS4Intents\
852 or "Error" in ONOS5Intents or not ONOS5Intents\
853 or "Error" in ONOS6Intents or not ONOS6Intents\
854 or "Error" in ONOS7Intents or not ONOS7Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -0800855 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800856 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
857 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
858 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
859 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
860 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
861 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
862 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
863 elif ONOS1Intents == ONOS2Intents\
864 and ONOS1Intents == ONOS3Intents\
865 and ONOS1Intents == ONOS4Intents\
866 and ONOS1Intents == ONOS5Intents\
867 and ONOS1Intents == ONOS6Intents\
868 and ONOS1Intents == ONOS7Intents:
869 intentState = ONOS1Intents
870 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800871 main.log.report( "Intents are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800872 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800873 main.log.warn(
874 "ONOS1 intents: ",
875 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800876 json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800877 sort_keys=True,
878 indent=4,
879 separators=(
880 ',',
881 ': ' ) ) )
882 main.log.warn(
883 "ONOS2 intents: ",
884 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800885 json.loads( ONOS2Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800886 sort_keys=True,
887 indent=4,
888 separators=(
889 ',',
890 ': ' ) ) )
891 main.log.warn(
892 "ONOS3 intents: ",
893 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800894 json.loads( ONOS3Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800895 sort_keys=True,
896 indent=4,
897 separators=(
898 ',',
899 ': ' ) ) )
900 main.log.warn(
901 "ONOS4 intents: ",
902 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800903 json.loads( ONOS4Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800904 sort_keys=True,
905 indent=4,
906 separators=(
907 ',',
908 ': ' ) ) )
909 main.log.warn(
910 "ONOS5 intents: ",
911 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800912 json.loads( ONOS5Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800913 sort_keys=True,
914 indent=4,
915 separators=(
916 ',',
917 ': ' ) ) )
918 main.log.warn(
919 "ONOS6 intents: ",
920 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800921 json.loads( ONOS6Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800922 sort_keys=True,
923 indent=4,
924 separators=(
925 ',',
926 ': ' ) ) )
927 main.log.warn(
928 "ONOS7 intents: ",
929 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800930 json.loads( ONOS7Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800931 sort_keys=True,
932 indent=4,
933 separators=(
934 ',',
935 ': ' ) ) )
936 utilities.assert_equals(
937 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800938 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800939 onpass="Intents are consistent across all ONOS nodes",
940 onfail="ONOS nodes have different views of intents" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800941
Jon Hall6aec96b2015-01-19 14:49:31 -0800942 main.step( "Get the flows from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800943 global flowState
944 flowState = []
Jon Hall8f89dda2015-01-22 16:03:33 -0800945 flowCheck = main.FALSE
Jon Hall58c76b72015-02-23 11:09:24 -0800946 try:
947 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
948 ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
949 ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
950 ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
951 ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
952 ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
953 ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
954 assert ONOS1Flows, "ONOS1 Flows should not be empty"
955 assert ONOS2Flows, "ONOS2 Flows should not be empty"
956 assert ONOS3Flows, "ONOS3 Flows should not be empty"
957 assert ONOS4Flows, "ONOS4 Flows should not be empty"
958 assert ONOS5Flows, "ONOS5 Flows should not be empty"
959 assert ONOS6Flows, "ONOS6 Flows should not be empty"
960 assert ONOS7Flows, "ONOS7 Flows should not be empty"
961 assert "Error" not in ONOS1Flows, "ONOS1 Flows contains 'Error'"
962 assert "Error" not in ONOS2Flows, "ONOS2 Flows contains 'Error'"
963 assert "Error" not in ONOS3Flows, "ONOS3 Flows contains 'Error'"
964 assert "Error" not in ONOS4Flows, "ONOS4 Flows contains 'Error'"
965 assert "Error" not in ONOS5Flows, "ONOS5 Flows contains 'Error'"
966 assert "Error" not in ONOS6Flows, "ONOS6 Flows contains 'Error'"
967 assert "Error" not in ONOS7Flows, "ONOS7 Flows contains 'Error'"
968 ONOS1FlowsJson = json.loads( ONOS1Flows )
969 ONOS2FlowsJson = json.loads( ONOS2Flows )
970 ONOS3FlowsJson = json.loads( ONOS3Flows )
971 ONOS4FlowsJson = json.loads( ONOS4Flows )
972 ONOS5FlowsJson = json.loads( ONOS5Flows )
973 ONOS6FlowsJson = json.loads( ONOS6Flows )
974 ONOS7FlowsJson = json.loads( ONOS7Flows )
975 except ( ValueError, AssertionError ): # From json.loads, or asserts
976 main.log.exception( "One or more 'flows' responses from " +
977 "ONOS couldn't be decoded." )
Jon Hall8f89dda2015-01-22 16:03:33 -0800978 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
979 main.log.warn( "ONOS2 flows repsponse: " + ONOS2Flows )
980 main.log.warn( "ONOS3 flows repsponse: " + ONOS3Flows )
981 main.log.warn( "ONOS4 flows repsponse: " + ONOS4Flows )
982 main.log.warn( "ONOS5 flows repsponse: " + ONOS5Flows )
983 main.log.warn( "ONOS6 flows repsponse: " + ONOS6Flows )
984 main.log.warn( "ONOS7 flows repsponse: " + ONOS7Flows )
Jon Hall58c76b72015-02-23 11:09:24 -0800985 else: # No exceptions
986 if len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
987 and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
988 and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
989 and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
990 and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
991 and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
Jon Hall6aec96b2015-01-19 14:49:31 -0800992 # TODO: Do a better check, maybe compare flows on switches?
Jon Hall58c76b72015-02-23 11:09:24 -0800993 # NOTE Possible issue with this not always being set?
994 flowState = ONOS1Flows
995 flowCheck = main.TRUE
996 main.log.report( "Flow count is consistent across all" +
997 " ONOS nodes" )
998 else:
999 main.log.warn( "ONOS1 flows: " +
1000 json.dumps( ONOS1FlowsJson, sort_keys=True,
1001 indent=4,
1002 separators=( ',', ': ' ) ) )
1003 main.log.warn( "ONOS2 flows: " +
1004 json.dumps( ONOS2FlowsJson, sort_keys=True,
1005 indent=4,
1006 separators=( ',', ': ' ) ) )
1007 main.log.warn( "ONOS3 flows: " +
1008 json.dumps( ONOS3FlowsJson, sort_keys=True,
1009 indent=4,
1010 separators=( ',', ': ' ) ) )
1011 main.log.warn( "ONOS4 flows: " +
1012 json.dumps( ONOS4FlowsJson, sort_keys=True,
1013 indent=4,
1014 separators=( ',', ': ' ) ) )
1015 main.log.warn( "ONOS5 flows: " +
1016 json.dumps( ONOS5FlowsJson, sort_keys=True,
1017 indent=4,
1018 separators=( ',', ': ' ) ) )
1019 main.log.warn( "ONOS6 flows: " +
1020 json.dumps( ONOS6FlowsJson, sort_keys=True,
1021 indent=4,
1022 separators=( ',', ': ' ) ) )
1023 main.log.warn( "ONOS7 flows: " +
1024 json.dumps( ONOS7FlowsJson, sort_keys=True,
1025 indent=4,
1026 separators=( ',', ': ' ) ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001027 utilities.assert_equals(
1028 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001029 actual=flowCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001030 onpass="The flow count is consistent across all ONOS nodes",
1031 onfail="ONOS nodes have different flow counts" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001032
Jon Hall6aec96b2015-01-19 14:49:31 -08001033 main.step( "Get the OF Table entries" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001034 global flows
Jon Hall6aec96b2015-01-19 14:49:31 -08001035 flows = []
1036 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001037 flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001038 if flowCheck == main.FALSE:
1039 for table in flows:
1040 main.log.warn( table )
Jon Hall6aec96b2015-01-19 14:49:31 -08001041 # TODO: Compare switch flow tables with ONOS flow tables
Jon Hall73cf9cc2014-11-20 22:28:38 -08001042
Jon Hall6aec96b2015-01-19 14:49:31 -08001043 main.step( "Start continuous pings" )
1044 main.Mininet2.pingLong(
1045 src=main.params[ 'PING' ][ 'source1' ],
1046 target=main.params[ 'PING' ][ 'target1' ],
1047 pingTime=500 )
1048 main.Mininet2.pingLong(
1049 src=main.params[ 'PING' ][ 'source2' ],
1050 target=main.params[ 'PING' ][ 'target2' ],
1051 pingTime=500 )
1052 main.Mininet2.pingLong(
1053 src=main.params[ 'PING' ][ 'source3' ],
1054 target=main.params[ 'PING' ][ 'target3' ],
1055 pingTime=500 )
1056 main.Mininet2.pingLong(
1057 src=main.params[ 'PING' ][ 'source4' ],
1058 target=main.params[ 'PING' ][ 'target4' ],
1059 pingTime=500 )
1060 main.Mininet2.pingLong(
1061 src=main.params[ 'PING' ][ 'source5' ],
1062 target=main.params[ 'PING' ][ 'target5' ],
1063 pingTime=500 )
1064 main.Mininet2.pingLong(
1065 src=main.params[ 'PING' ][ 'source6' ],
1066 target=main.params[ 'PING' ][ 'target6' ],
1067 pingTime=500 )
1068 main.Mininet2.pingLong(
1069 src=main.params[ 'PING' ][ 'source7' ],
1070 target=main.params[ 'PING' ][ 'target7' ],
1071 pingTime=500 )
1072 main.Mininet2.pingLong(
1073 src=main.params[ 'PING' ][ 'source8' ],
1074 target=main.params[ 'PING' ][ 'target8' ],
1075 pingTime=500 )
1076 main.Mininet2.pingLong(
1077 src=main.params[ 'PING' ][ 'source9' ],
1078 target=main.params[ 'PING' ][ 'target9' ],
1079 pingTime=500 )
1080 main.Mininet2.pingLong(
1081 src=main.params[ 'PING' ][ 'source10' ],
1082 target=main.params[ 'PING' ][ 'target10' ],
1083 pingTime=500 )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001084
Jon Hall6aec96b2015-01-19 14:49:31 -08001085 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001086 ctrls = []
1087 count = 1
1088 while True:
1089 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -08001090 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
1091 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
1092 temp = temp + ( "ONOS" + str( count ), )
1093 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
1094 temp = temp + \
1095 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
1096 ctrls.append( temp )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001097 count = count + 1
1098 else:
1099 break
Jon Hall6aec96b2015-01-19 14:49:31 -08001100 MNTopo = TestONTopology(
1101 main.Mininet1,
1102 ctrls ) # can also add Intent API info for intent operations
Jon Hall73cf9cc2014-11-20 22:28:38 -08001103
Jon Hall6aec96b2015-01-19 14:49:31 -08001104 main.step( "Collecting topology information from ONOS" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001105 devices = []
1106 devices.append( main.ONOScli1.devices() )
1107 devices.append( main.ONOScli2.devices() )
1108 devices.append( main.ONOScli3.devices() )
1109 devices.append( main.ONOScli4.devices() )
1110 devices.append( main.ONOScli5.devices() )
1111 devices.append( main.ONOScli6.devices() )
1112 devices.append( main.ONOScli7.devices() )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001113 hosts = []
Jon Hall58c76b72015-02-23 11:09:24 -08001114 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1115 hosts.append( json.loads( main.ONOScli2.hosts() ) )
1116 hosts.append( json.loads( main.ONOScli3.hosts() ) )
1117 hosts.append( json.loads( main.ONOScli4.hosts() ) )
1118 hosts.append( json.loads( main.ONOScli5.hosts() ) )
1119 hosts.append( json.loads( main.ONOScli6.hosts() ) )
1120 hosts.append( json.loads( main.ONOScli7.hosts() ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001121 ports = []
1122 ports.append( main.ONOScli1.ports() )
1123 ports.append( main.ONOScli2.ports() )
1124 ports.append( main.ONOScli3.ports() )
1125 ports.append( main.ONOScli4.ports() )
1126 ports.append( main.ONOScli5.ports() )
1127 ports.append( main.ONOScli6.ports() )
1128 ports.append( main.ONOScli7.ports() )
1129 links = []
1130 links.append( main.ONOScli1.links() )
1131 links.append( main.ONOScli2.links() )
1132 links.append( main.ONOScli3.links() )
1133 links.append( main.ONOScli4.links() )
1134 links.append( main.ONOScli5.links() )
1135 links.append( main.ONOScli6.links() )
1136 links.append( main.ONOScli7.links() )
Jon Hall94fd0472014-12-08 11:52:42 -08001137 clusters = []
1138 clusters.append( main.ONOScli1.clusters() )
1139 clusters.append( main.ONOScli2.clusters() )
1140 clusters.append( main.ONOScli3.clusters() )
1141 clusters.append( main.ONOScli4.clusters() )
1142 clusters.append( main.ONOScli5.clusters() )
1143 clusters.append( main.ONOScli6.clusters() )
1144 clusters.append( main.ONOScli7.clusters() )
Jon Hall529a37f2015-01-28 10:02:00 -08001145 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001146
Jon Hall6aec96b2015-01-19 14:49:31 -08001147 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -08001148 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001149 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001150 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001151 if "Error" not in hosts[ controller ]:
1152 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001153 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001154 else: # hosts not consistent
1155 main.log.report( "hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001156 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001157 " is inconsistent with ONOS1" )
1158 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001159 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001160
1161 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001162 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001163 controllerStr )
1164 consistentHostsResult = main.FALSE
1165 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001166 " hosts response: " +
1167 repr( hosts[ controller ] ) )
1168 utilities.assert_equals(
1169 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001170 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001171 onpass="Hosts view is consistent across all ONOS nodes",
1172 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001173
Jon Hall58c76b72015-02-23 11:09:24 -08001174 ipResult = main.TRUE
1175 for controller in range( 0, len( hosts ) ):
1176 controllerStr = str( controller + 1 )
1177 for host in hosts[ controller ]:
1178 if host.get( 'ips', [] ) == []:
1179 main.log.error(
1180 "DEBUG:Error with host ips on controller" +
1181 controllerStr + ": " + str( host ) )
1182 ipResult = main.FALSE
1183 utilities.assert_equals(
1184 expect=main.TRUE,
1185 actual=ipResult,
1186 onpass="The ips of the hosts aren't empty",
1187 onfail="The ip of at least one host is missing" )
1188
Jon Hall6aec96b2015-01-19 14:49:31 -08001189 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001190 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001191 for controller in range( len( clusters ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001192 if "Error" not in clusters[ controller ]:
1193 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001194 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001195 else: # clusters not consistent
1196 main.log.report( "clusters from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001197 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001198 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001199 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001200
1201 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001202 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001203 "from ONOS" + controllerStr )
1204 consistentClustersResult = main.FALSE
1205 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001206 " clusters response: " +
1207 repr( clusters[ controller ] ) )
1208 utilities.assert_equals(
1209 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001210 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001211 onpass="Clusters view is consistent across all ONOS nodes",
1212 onfail="ONOS nodes have different views of clusters" )
1213 # there should always only be one cluster
Jon Hall8f89dda2015-01-22 16:03:33 -08001214 numClusters = len( json.loads( clusters[ 0 ] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001215 clusterResults = main.FALSE
1216 if numClusters == 1:
1217 clusterResults = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001218 utilities.assert_equals(
1219 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001220 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001221 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08001222 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001223
Jon Hall6aec96b2015-01-19 14:49:31 -08001224 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001225 devicesResults = main.TRUE
1226 portsResults = main.TRUE
1227 linksResults = main.TRUE
1228 for controller in range( numControllers ):
1229 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001230 if devices[ controller ] or "Error" not in devices[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001231 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001232 MNTopo,
1233 json.loads(
1234 devices[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001235 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001236 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001237 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001238 actual=currentDevicesResult,
1239 onpass="ONOS" + controllerStr +
1240 " Switches view is correct",
1241 onfail="ONOS" + controllerStr +
1242 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001243
Jon Hall6aec96b2015-01-19 14:49:31 -08001244 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001245 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001246 MNTopo,
1247 json.loads(
1248 ports[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001249 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001250 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001251 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001252 actual=currentPortsResult,
1253 onpass="ONOS" + controllerStr +
1254 " ports view is correct",
1255 onfail="ONOS" + controllerStr +
1256 " ports view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001257
Jon Hall6aec96b2015-01-19 14:49:31 -08001258 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001259 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001260 MNTopo,
1261 json.loads(
1262 links[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001263 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001264 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001265 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001266 actual=currentLinksResult,
1267 onpass="ONOS" + controllerStr +
1268 " links view is correct",
1269 onfail="ONOS" + controllerStr +
1270 " links view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001271
Jon Hall8f89dda2015-01-22 16:03:33 -08001272 devicesResults = devicesResults and currentDevicesResult
1273 portsResults = portsResults and currentPortsResult
1274 linksResults = linksResults and currentLinksResult
Jon Hall73cf9cc2014-11-20 22:28:38 -08001275
Jon Hall8f89dda2015-01-22 16:03:33 -08001276 topoResult = devicesResults and portsResults and linksResults\
Jon Hall58c76b72015-02-23 11:09:24 -08001277 and consistentHostsResult and consistentClustersResult\
1278 and clusterResults and ipResult
Jon Hall8f89dda2015-01-22 16:03:33 -08001279 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08001280 onpass="Topology Check Test successful",
1281 onfail="Topology Check Test NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001282
Jon Hall8f89dda2015-01-22 16:03:33 -08001283 finalAssert = main.TRUE
1284 finalAssert = finalAssert and topoResult and flowCheck \
Jon Hall58c76b72015-02-23 11:09:24 -08001285 and intentCheck and consistentMastership and rolesNotNull
Jon Hall8f89dda2015-01-22 16:03:33 -08001286 utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
Jon Hall58c76b72015-02-23 11:09:24 -08001287 onpass="State check successful",
1288 onfail="State check NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001289
Jon Hall6aec96b2015-01-19 14:49:31 -08001290 def CASE6( self, main ):
1291 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001292 The Failure case.
Jon Hall6aec96b2015-01-19 14:49:31 -08001293 """
Jon Hall94fd0472014-12-08 11:52:42 -08001294 import time
Jon Hall6aec96b2015-01-19 14:49:31 -08001295 main.log.report( "Killing 3 ONOS nodes" )
1296 main.log.case( "Restart minority of ONOS nodes" )
1297 # TODO: Randomize these nodes
Jon Hall8f89dda2015-01-22 16:03:33 -08001298 main.ONOSbench.onosKill( ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -08001299 time.sleep( 10 )
Jon Hall8f89dda2015-01-22 16:03:33 -08001300 main.ONOSbench.onosKill( ONOS2Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -08001301 time.sleep( 10 )
Jon Hall8f89dda2015-01-22 16:03:33 -08001302 main.ONOSbench.onosKill( ONOS3Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001303
Jon Hall6aec96b2015-01-19 14:49:31 -08001304 main.step( "Checking if ONOS is up yet" )
Jon Hallffb386d2014-11-21 13:43:38 -08001305 count = 0
Jon Hall8f89dda2015-01-22 16:03:33 -08001306 onosIsupResult = main.FALSE
1307 while onosIsupResult == main.FALSE and count < 10:
1308 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
1309 onos2Isup = main.ONOSbench.isup( ONOS2Ip )
1310 onos3Isup = main.ONOSbench.isup( ONOS3Ip )
1311 onosIsupResult = onos1Isup and onos2Isup and onos3Isup
Jon Hallffb386d2014-11-21 13:43:38 -08001312 count = count + 1
Jon Hall73cf9cc2014-11-20 22:28:38 -08001313 # TODO: if it becomes an issue, we can retry this step a few times
1314
Jon Hall8f89dda2015-01-22 16:03:33 -08001315 cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
1316 cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
1317 cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
1318 cliResults = cliResult1 and cliResult2 and cliResult3
Jon Hall73cf9cc2014-11-20 22:28:38 -08001319
Jon Hall21270ac2015-02-16 17:59:55 -08001320 # Grab the time of restart so we chan check how long the gossip
1321 # protocol has had time to work
1322 main.restartTime = time.time()
Jon Hall8f89dda2015-01-22 16:03:33 -08001323 caseResults = main.TRUE and onosIsupResult and cliResults
1324 utilities.assert_equals( expect=main.TRUE, actual=caseResults,
Jon Hall58c76b72015-02-23 11:09:24 -08001325 onpass="ONOS restart successful",
1326 onfail="ONOS restart NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001327
Jon Hall6aec96b2015-01-19 14:49:31 -08001328 def CASE7( self, main ):
1329 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001330 Check state after ONOS failure
Jon Hall6aec96b2015-01-19 14:49:31 -08001331 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001332 import json
Jon Hallfebb1c72015-03-05 13:30:09 -08001333 import time
Jon Hall6aec96b2015-01-19 14:49:31 -08001334 main.case( "Running ONOS Constant State Tests" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001335
Jon Hall6aec96b2015-01-19 14:49:31 -08001336 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -08001337 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
1338 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
1339 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
1340 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
1341 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
1342 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
1343 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
1344 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
1345 ONOS3MasterNotNull and ONOS4MasterNotNull and\
1346 ONOS5MasterNotNull and ONOS6MasterNotNull and\
1347 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -08001348 utilities.assert_equals(
1349 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001350 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -08001351 onpass="Each device has a master",
1352 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -08001353
Jon Hall6aec96b2015-01-19 14:49:31 -08001354 main.step( "Check if switch roles are consistent across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001355 ONOS1Mastership = main.ONOScli1.roles()
1356 ONOS2Mastership = main.ONOScli2.roles()
1357 ONOS3Mastership = main.ONOScli3.roles()
1358 ONOS4Mastership = main.ONOScli4.roles()
1359 ONOS5Mastership = main.ONOScli5.roles()
1360 ONOS6Mastership = main.ONOScli6.roles()
1361 ONOS7Mastership = main.ONOScli7.roles()
Jon Hall8f89dda2015-01-22 16:03:33 -08001362 if "Error" in ONOS1Mastership or not ONOS1Mastership\
1363 or "Error" in ONOS2Mastership or not ONOS2Mastership\
1364 or "Error" in ONOS3Mastership or not ONOS3Mastership\
1365 or "Error" in ONOS4Mastership or not ONOS4Mastership\
1366 or "Error" in ONOS5Mastership or not ONOS5Mastership\
1367 or "Error" in ONOS6Mastership or not ONOS6Mastership\
1368 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -08001369 main.log.error( "Error in getting ONOS mastership" )
1370 main.log.warn( "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001371 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001372 main.log.warn( "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001373 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001374 main.log.warn( "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001375 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001376 main.log.warn( "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001377 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001378 main.log.warn( "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001379 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001380 main.log.warn( "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001381 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001382 main.log.warn( "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001383 repr( ONOS7Mastership ) )
1384 consistentMastership = main.FALSE
1385 elif ONOS1Mastership == ONOS2Mastership\
1386 and ONOS1Mastership == ONOS3Mastership\
1387 and ONOS1Mastership == ONOS4Mastership\
1388 and ONOS1Mastership == ONOS5Mastership\
1389 and ONOS1Mastership == ONOS6Mastership\
1390 and ONOS1Mastership == ONOS7Mastership:
1391 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001392 main.log.report(
1393 "Switch roles are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001394 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001395 main.log.warn( "ONOS1 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001396 json.loads( ONOS1Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001397 separators=( ',', ': ' ) ) )
1398 main.log.warn( "ONOS2 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001399 json.loads( ONOS2Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001400 separators=( ',', ': ' ) ) )
1401 main.log.warn( "ONOS3 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001402 json.loads( ONOS3Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001403 separators=( ',', ': ' ) ) )
1404 main.log.warn( "ONOS4 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001405 json.loads( ONOS4Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001406 separators=( ',', ': ' ) ) )
1407 main.log.warn( "ONOS5 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001408 json.loads( ONOS5Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001409 separators=( ',', ': ' ) ) )
1410 main.log.warn( "ONOS6 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001411 json.loads( ONOS6Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001412 separators=( ',', ': ' ) ) )
1413 main.log.warn( "ONOS7 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001414 json.loads( ONOS7Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001415 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001416 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001417 utilities.assert_equals(
1418 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001419 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -08001420 onpass="Switch roles are consistent across all ONOS nodes",
1421 onfail="ONOS nodes have different views of switch roles" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001422
1423 description2 = "Compare switch roles from before failure"
Jon Hall6aec96b2015-01-19 14:49:31 -08001424 main.step( description2 )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001425
Jon Hall8f89dda2015-01-22 16:03:33 -08001426 currentJson = json.loads( ONOS1Mastership )
1427 oldJson = json.loads( mastershipState )
1428 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001429 for i in range( 1, 29 ):
1430 switchDPID = str(
1431 main.Mininet1.getSwitchDPID(
1432 switch="s" +
1433 str( i ) ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001434
Jon Hall8f89dda2015-01-22 16:03:33 -08001435 current = [ switch[ 'master' ] for switch in currentJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001436 if switchDPID in switch[ 'id' ] ]
Jon Hall8f89dda2015-01-22 16:03:33 -08001437 old = [ switch[ 'master' ] for switch in oldJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001438 if switchDPID in switch[ 'id' ] ]
Jon Hall73cf9cc2014-11-20 22:28:38 -08001439 if current == old:
Jon Hall8f89dda2015-01-22 16:03:33 -08001440 mastershipCheck = mastershipCheck and main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001441 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001442 main.log.warn( "Mastership of switch %s changed" % switchDPID )
Jon Hall8f89dda2015-01-22 16:03:33 -08001443 mastershipCheck = main.FALSE
1444 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001445 main.log.report( "Mastership of Switches was not changed" )
1446 utilities.assert_equals(
1447 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001448 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001449 onpass="Mastership of Switches was not changed",
1450 onfail="Mastership of some switches changed" )
1451 # NOTE: we expect mastership to change on controller failure
Jon Hall8f89dda2015-01-22 16:03:33 -08001452 mastershipCheck = consistentMastership
Jon Hall73cf9cc2014-11-20 22:28:38 -08001453
Jon Hall58c76b72015-02-23 11:09:24 -08001454 main.step( "Get the intents and compare across all nodes" )
1455 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1456 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
1457 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
1458 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
1459 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
1460 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
1461 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
1462 intentCheck = main.FALSE
1463 if "Error" in ONOS1Intents or not ONOS1Intents\
1464 or "Error" in ONOS2Intents or not ONOS2Intents\
1465 or "Error" in ONOS3Intents or not ONOS3Intents\
1466 or "Error" in ONOS4Intents or not ONOS4Intents\
1467 or "Error" in ONOS5Intents or not ONOS5Intents\
1468 or "Error" in ONOS6Intents or not ONOS6Intents\
1469 or "Error" in ONOS7Intents or not ONOS7Intents:
1470 main.log.report( "Error in getting ONOS intents" )
1471 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1472 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
1473 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
1474 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
1475 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
1476 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
1477 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
1478 elif ONOS1Intents == ONOS2Intents\
1479 and ONOS1Intents == ONOS3Intents\
1480 and ONOS1Intents == ONOS4Intents\
1481 and ONOS1Intents == ONOS5Intents\
1482 and ONOS1Intents == ONOS6Intents\
1483 and ONOS1Intents == ONOS7Intents:
1484 intentCheck = main.TRUE
1485 main.log.report( "Intents are consistent across all ONOS nodes" )
1486 else:
1487 main.log.warn( "ONOS1 intents: " )
1488 print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
1489 indent=4, separators=( ',', ': ' ) )
1490 main.log.warn( "ONOS2 intents: " )
1491 print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
1492 indent=4, separators=( ',', ': ' ) )
1493 main.log.warn( "ONOS3 intents: " )
1494 print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
1495 indent=4, separators=( ',', ': ' ) )
1496 main.log.warn( "ONOS4 intents: " )
1497 print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
1498 indent=4, separators=( ',', ': ' ) )
1499 main.log.warn( "ONOS5 intents: " )
1500 print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
1501 indent=4, separators=( ',', ': ' ) )
1502 main.log.warn( "ONOS6 intents: " )
1503 print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
1504 indent=4, separators=( ',', ': ' ) )
1505 main.log.warn( "ONOS7 intents: " )
1506 print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
1507 indent=4, separators=( ',', ': ' ) )
1508 utilities.assert_equals(
1509 expect=main.TRUE,
1510 actual=intentCheck,
1511 onpass="Intents are consistent across all ONOS nodes",
1512 onfail="ONOS nodes have different views of intents" )
1513 # Print the intent states
1514 intents = []
1515 intents.append( ONOS1Intents )
1516 intents.append( ONOS2Intents )
1517 intents.append( ONOS3Intents )
1518 intents.append( ONOS4Intents )
1519 intents.append( ONOS5Intents )
1520 intents.append( ONOS6Intents )
1521 intents.append( ONOS7Intents )
1522 intentStates = []
1523 for node in intents: # Iter through ONOS nodes
1524 nodeStates = []
1525 # Iter through intents of a node
1526 for intent in json.loads( node ):
1527 nodeStates.append( intent[ 'state' ] )
1528 intentStates.append( nodeStates )
1529 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1530 main.log.info( dict( out ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001531
Jon Hall58c76b72015-02-23 11:09:24 -08001532 # NOTE: Store has no durability, so intents are lost across system
1533 # restarts
1534 main.step( "Compare current intents with intents before the failure" )
1535 # NOTE: this requires case 5 to pass for intentState to be set.
1536 # maybe we should stop the test if that fails?
1537 sameIntents = main.TRUE
1538 if intentState and intentState == ONOS1Intents:
Jon Hall21270ac2015-02-16 17:59:55 -08001539 sameIntents = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -08001540 main.log.report( "Intents are consistent with before failure" )
1541 # TODO: possibly the states have changed? we may need to figure out
1542 # what the aceptable states are
1543 else:
1544 try:
1545 main.log.warn( "ONOS1 intents: " )
1546 print json.dumps( json.loads( ONOS1Intents ),
1547 sort_keys=True, indent=4,
1548 separators=( ',', ': ' ) )
Jon Hall88e498c2015-03-06 09:54:35 -08001549 except Exception:
Jon Hall58c76b72015-02-23 11:09:24 -08001550 pass
1551 sameIntents = main.FALSE
1552 utilities.assert_equals(
1553 expect=main.TRUE,
1554 actual=sameIntents,
1555 onpass="Intents are consistent with before failure",
1556 onfail="The Intents changed during failure" )
1557 intentCheck = intentCheck and sameIntents
Jon Hall21270ac2015-02-16 17:59:55 -08001558
Jon Hall58c76b72015-02-23 11:09:24 -08001559 main.step( "Get the OF Table entries and compare to before " +
1560 "component failure" )
1561 FlowTables = main.TRUE
1562 flows2 = []
1563 for i in range( 28 ):
1564 main.log.info( "Checking flow table on s" + str( i + 1 ) )
1565 tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
1566 flows2.append( tmpFlows )
1567 tempResult = main.Mininet2.flowComp(
1568 flow1=flows[ i ],
1569 flow2=tmpFlows )
1570 FlowTables = FlowTables and tempResult
1571 if FlowTables == main.FALSE:
1572 main.log.info( "Differences in flow table for switch: s" +
1573 str( i + 1 ) )
1574 if FlowTables == main.TRUE:
1575 main.log.report( "No changes were found in the flow tables" )
1576 utilities.assert_equals(
1577 expect=main.TRUE,
1578 actual=FlowTables,
1579 onpass="No changes were found in the flow tables",
1580 onfail="Changes were found in the flow tables" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001581
Jon Hall6aec96b2015-01-19 14:49:31 -08001582 main.step( "Check the continuous pings to ensure that no packets " +
1583 "were dropped during component failure" )
1584 # FIXME: This check is always failing. Investigate cause
1585 # NOTE: this may be something to do with file permsissions
Jon Hall73cf9cc2014-11-20 22:28:38 -08001586 # or slight change in format
Jon Hall6aec96b2015-01-19 14:49:31 -08001587 main.Mininet2.pingKill(
1588 main.params[ 'TESTONUSER' ],
1589 main.params[ 'TESTONIP' ] )
Jon Hall8f89dda2015-01-22 16:03:33 -08001590 LossInPings = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001591 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1592 for i in range( 8, 18 ):
1593 main.log.info(
1594 "Checking for a loss in pings along flow from s" +
1595 str( i ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001596 LossInPings = main.Mininet2.checkForLoss(
Jon Hall6aec96b2015-01-19 14:49:31 -08001597 "/tmp/ping.h" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001598 str( i ) ) or LossInPings
1599 if LossInPings == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001600 main.log.info( "Loss in ping detected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001601 elif LossInPings == main.ERROR:
Jon Hall6aec96b2015-01-19 14:49:31 -08001602 main.log.info( "There are multiple mininet process running" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001603 elif LossInPings == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001604 main.log.info( "No Loss in the pings" )
1605 main.log.report( "No loss of dataplane connectivity" )
1606 utilities.assert_equals(
1607 expect=main.FALSE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001608 actual=LossInPings,
Jon Hall6aec96b2015-01-19 14:49:31 -08001609 onpass="No Loss of connectivity",
1610 onfail="Loss of dataplane connectivity detected" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001611
Jon Hall6aec96b2015-01-19 14:49:31 -08001612 # Test of LeadershipElection
Jon Hall8f89dda2015-01-22 16:03:33 -08001613 leaderList = []
1614 leaderResult = main.TRUE
1615 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001616 # loop through ONOScli handlers
1617 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001618 leaderN = node.electionTestLeader()
1619 leaderList.append( leaderN )
Jon Hall669173b2014-12-17 11:36:30 -08001620 if leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001621 # error in response
1622 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001623 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001624 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001625 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001626 elif leaderN is None:
1627 main.log.report( "ONOS" + str( controller ) +
1628 " shows no leader for the election-app was" +
1629 " elected after the old one died" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001630 leaderResult = main.FALSE
1631 elif leaderN == ONOS1Ip or leaderN == ONOS2Ip or\
1632 leaderN == ONOS3Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08001633 main.log.report( "ONOS" + str( controller ) +
1634 " shows " + str( leaderN ) +
1635 " as leader for the election-app, but it " +
1636 "was restarted" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001637 leaderResult = main.FALSE
1638 if len( set( leaderList ) ) != 1:
1639 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001640 main.log.error(
1641 "Inconsistent view of leader for the election test app" )
1642 # TODO: print the list
Jon Hall8f89dda2015-01-22 16:03:33 -08001643 if leaderResult:
1644 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001645 "view of leader across listeners and a new " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001646 "leader was re-elected if applicable )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001647 utilities.assert_equals(
1648 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001649 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001650 onpass="Leadership election passed",
1651 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08001652
Jon Hall58c76b72015-02-23 11:09:24 -08001653 result = ( mastershipCheck and intentCheck and FlowTables and
1654 ( not LossInPings ) and rolesNotNull and leaderResult )
Jon Hall6aec96b2015-01-19 14:49:31 -08001655 result = int( result )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001656 if result == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001657 main.log.report( "Constant State Tests Passed" )
1658 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hallfebb1c72015-03-05 13:30:09 -08001659 onpass="Constant State Tests Passed",
1660 onfail="Constant state tests failed" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001661
Jon Hall6aec96b2015-01-19 14:49:31 -08001662 def CASE8( self, main ):
1663 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001664 Compare topo
Jon Hall6aec96b2015-01-19 14:49:31 -08001665 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001666 import sys
Jon Hall6aec96b2015-01-19 14:49:31 -08001667 # FIXME add this path to params
1668 sys.path.append( "/home/admin/sts" )
1669 # assumes that sts is already in you PYTHONPATH
1670 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -08001671 import json
1672 import time
1673
Jon Hall6aec96b2015-01-19 14:49:31 -08001674 description = "Compare ONOS Topology view to Mininet topology"
1675 main.case( description )
1676 main.log.report( description )
1677 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001678 ctrls = []
1679 count = 1
1680 while True:
1681 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -08001682 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
1683 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
1684 temp = temp + ( "ONOS" + str( count ), )
1685 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
1686 temp = temp + \
1687 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
1688 ctrls.append( temp )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001689 count = count + 1
1690 else:
1691 break
Jon Hall6aec96b2015-01-19 14:49:31 -08001692 MNTopo = TestONTopology(
1693 main.Mininet1,
1694 ctrls ) # can also add Intent API info for intent operations
Jon Hall73cf9cc2014-11-20 22:28:38 -08001695
Jon Hall6aec96b2015-01-19 14:49:31 -08001696 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001697 devicesResults = main.TRUE
1698 portsResults = main.TRUE
1699 linksResults = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -08001700 hostsResults = main.TRUE
Jon Hall8f89dda2015-01-22 16:03:33 -08001701 topoResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001702 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -08001703 count = 0
Jon Hall6aec96b2015-01-19 14:49:31 -08001704 main.step( "Collecting topology information from ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001705 startTime = time.time()
Jon Hall21270ac2015-02-16 17:59:55 -08001706 # Give time for Gossip to work
Jon Hall8f89dda2015-01-22 16:03:33 -08001707 while topoResult == main.FALSE and elapsed < 60:
Jon Hallffb386d2014-11-21 13:43:38 -08001708 count = count + 1
Jon Hall94fd0472014-12-08 11:52:42 -08001709 if count > 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08001710 # TODO: Depricate STS usage
Jon Hall58c76b72015-02-23 11:09:24 -08001711 MNTopo = TestONTopology( main.Mininet1, ctrls )
Jon Hall8f89dda2015-01-22 16:03:33 -08001712 cliStart = time.time()
Jon Hall94fd0472014-12-08 11:52:42 -08001713 devices = []
1714 devices.append( main.ONOScli1.devices() )
1715 devices.append( main.ONOScli2.devices() )
1716 devices.append( main.ONOScli3.devices() )
1717 devices.append( main.ONOScli4.devices() )
1718 devices.append( main.ONOScli5.devices() )
1719 devices.append( main.ONOScli6.devices() )
1720 devices.append( main.ONOScli7.devices() )
1721 hosts = []
Jon Hall669173b2014-12-17 11:36:30 -08001722 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1723 hosts.append( json.loads( main.ONOScli2.hosts() ) )
1724 hosts.append( json.loads( main.ONOScli3.hosts() ) )
1725 hosts.append( json.loads( main.ONOScli4.hosts() ) )
1726 hosts.append( json.loads( main.ONOScli5.hosts() ) )
1727 hosts.append( json.loads( main.ONOScli6.hosts() ) )
1728 hosts.append( json.loads( main.ONOScli7.hosts() ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001729 ipResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001730 for controller in range( 0, len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001731 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001732 for host in hosts[ controller ]:
Jon Hall58c76b72015-02-23 11:09:24 -08001733 if host is None or host.get( 'ips', [] ) == []:
Jon Hall6aec96b2015-01-19 14:49:31 -08001734 main.log.error(
1735 "DEBUG:Error with host ips on controller" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001736 controllerStr + ": " + str( host ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001737 ipResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001738 ports = []
1739 ports.append( main.ONOScli1.ports() )
1740 ports.append( main.ONOScli2.ports() )
1741 ports.append( main.ONOScli3.ports() )
1742 ports.append( main.ONOScli4.ports() )
1743 ports.append( main.ONOScli5.ports() )
1744 ports.append( main.ONOScli6.ports() )
1745 ports.append( main.ONOScli7.ports() )
1746 links = []
1747 links.append( main.ONOScli1.links() )
1748 links.append( main.ONOScli2.links() )
1749 links.append( main.ONOScli3.links() )
1750 links.append( main.ONOScli4.links() )
1751 links.append( main.ONOScli5.links() )
1752 links.append( main.ONOScli6.links() )
1753 links.append( main.ONOScli7.links() )
1754 clusters = []
1755 clusters.append( main.ONOScli1.clusters() )
1756 clusters.append( main.ONOScli2.clusters() )
1757 clusters.append( main.ONOScli3.clusters() )
1758 clusters.append( main.ONOScli4.clusters() )
1759 clusters.append( main.ONOScli5.clusters() )
1760 clusters.append( main.ONOScli6.clusters() )
1761 clusters.append( main.ONOScli7.clusters() )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001762
Jon Hall8f89dda2015-01-22 16:03:33 -08001763 elapsed = time.time() - startTime
1764 cliTime = time.time() - cliStart
1765 print "CLI time: " + str( cliTime )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001766
Jon Hall8f89dda2015-01-22 16:03:33 -08001767 for controller in range( numControllers ):
1768 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001769 if devices[ controller ] or "Error" not in devices[
1770 controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001771 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001772 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001773 json.loads( devices[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001774 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001775 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001776 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001777 actual=currentDevicesResult,
1778 onpass="ONOS" + controllerStr +
1779 " Switches view is correct",
1780 onfail="ONOS" + controllerStr +
1781 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001782
Jon Hall6aec96b2015-01-19 14:49:31 -08001783 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001784 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001785 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001786 json.loads( ports[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001787 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001788 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001789 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001790 actual=currentPortsResult,
1791 onpass="ONOS" + controllerStr +
1792 " ports view is correct",
1793 onfail="ONOS" + controllerStr +
1794 " ports view is incorrect" )
Jon Hall94fd0472014-12-08 11:52:42 -08001795
Jon Hall6aec96b2015-01-19 14:49:31 -08001796 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001797 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001798 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001799 json.loads( links[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001800 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001801 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001802 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001803 actual=currentLinksResult,
1804 onpass="ONOS" + controllerStr +
1805 " links view is correct",
1806 onfail="ONOS" + controllerStr +
1807 " links view is incorrect" )
1808
1809 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1810 currentHostsResult = main.Mininet1.compareHosts(
1811 MNTopo, hosts[ controller ] )
1812 else:
1813 currentHostsResult = main.FALSE
1814 utilities.assert_equals( expect=main.TRUE,
1815 actual=currentHostsResult,
1816 onpass="ONOS" + controllerStr +
1817 " hosts exist in Mininet",
1818 onfail="ONOS" + controllerStr +
1819 " hosts don't match Mininet" )
1820
1821 devicesResults = devicesResults and currentDevicesResult
1822 portsResults = portsResults and currentPortsResult
1823 linksResults = linksResults and currentLinksResult
1824 hostsResults = hostsResults and currentHostsResult
Jon Hall94fd0472014-12-08 11:52:42 -08001825
Jon Hall529a37f2015-01-28 10:02:00 -08001826 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001827
Jon Hall6aec96b2015-01-19 14:49:31 -08001828 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -08001829 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001830 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001831 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001832 if "Error" not in hosts[ controller ]:
1833 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001834 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001835 else: # hosts not consistent
Jon Hall8f89dda2015-01-22 16:03:33 -08001836 main.log.report( "hosts from ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001837 " is inconsistent with ONOS1" )
1838 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001839 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001840
1841 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001842 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001843 controllerStr )
1844 consistentHostsResult = main.FALSE
1845 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001846 " hosts response: " +
1847 repr( hosts[ controller ] ) )
1848 utilities.assert_equals(
1849 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001850 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001851 onpass="Hosts view is consistent across all ONOS nodes",
1852 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001853
Jon Hall6aec96b2015-01-19 14:49:31 -08001854 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001855 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001856 for controller in range( len( clusters ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001857 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001858 if "Error" not in clusters[ controller ]:
1859 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001860 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001861 else: # clusters not consistent
1862 main.log.report( "clusters from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001863 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001864 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001865 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001866
1867 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001868 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001869 "from ONOS" + controllerStr )
1870 consistentClustersResult = main.FALSE
1871 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001872 " clusters response: " +
1873 repr( clusters[ controller ] ) )
1874 utilities.assert_equals(
1875 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001876 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001877 onpass="Clusters view is consistent across all ONOS nodes",
1878 onfail="ONOS nodes have different views of clusters" )
1879 # there should always only be one cluster
Jon Hall8f89dda2015-01-22 16:03:33 -08001880 numClusters = len( json.loads( clusters[ 0 ] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001881 clusterResults = main.FALSE
1882 if numClusters == 1:
1883 clusterResults = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001884 utilities.assert_equals(
1885 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001886 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001887 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08001888 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001889
Jon Hall8f89dda2015-01-22 16:03:33 -08001890 topoResult = ( devicesResults and portsResults and linksResults
Jon Hall58c76b72015-02-23 11:09:24 -08001891 and hostsResults and consistentHostsResult
1892 and consistentClustersResult and clusterResults
1893 and ipResult )
Jon Hall94fd0472014-12-08 11:52:42 -08001894
Jon Hall8f89dda2015-01-22 16:03:33 -08001895 topoResult = topoResult and int( count <= 2 )
1896 note = "note it takes about " + str( int( cliTime ) ) + \
1897 " seconds for the test to make all the cli calls to fetch " +\
1898 "the topology from each ONOS instance"
Jon Hall1b8f54a2015-02-04 13:24:20 -08001899 main.log.info(
Jon Hall8f89dda2015-01-22 16:03:33 -08001900 "Very crass estimate for topology discovery/convergence( " +
1901 str( note ) + " ): " + str( elapsed ) + " seconds, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001902 str( count ) + " tries" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001903 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08001904 onpass="Topology Check Test successful",
1905 onfail="Topology Check Test NOT successful" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001906 if topoResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001907 main.log.report( "ONOS topology view matches Mininet topology" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001908
Jon Hall6aec96b2015-01-19 14:49:31 -08001909 def CASE9( self, main ):
1910 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001911 Link s3-s28 down
Jon Hall6aec96b2015-01-19 14:49:31 -08001912 """
1913 import time
1914 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001915
Jon Hall8f89dda2015-01-22 16:03:33 -08001916 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001917
Jon Hall6aec96b2015-01-19 14:49:31 -08001918 description = "Turn off a link to ensure that Link Discovery " +\
Jon Hall58c76b72015-02-23 11:09:24 -08001919 "is working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001920 main.log.report( description )
1921 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001922
Jon Hall6aec96b2015-01-19 14:49:31 -08001923 main.step( "Kill Link between s3 and s28" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001924 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
Jon Hall58c76b72015-02-23 11:09:24 -08001925 main.log.info( "Waiting " + str( linkSleep ) +
1926 " seconds for link down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001927 time.sleep( linkSleep )
1928 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
Jon Hall58c76b72015-02-23 11:09:24 -08001929 onpass="Link down succesful",
1930 onfail="Failed to bring link down" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001931 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -08001932
Jon Hall6aec96b2015-01-19 14:49:31 -08001933 def CASE10( self, main ):
1934 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001935 Link s3-s28 up
Jon Hall6aec96b2015-01-19 14:49:31 -08001936 """
1937 import time
1938 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001939
Jon Hall8f89dda2015-01-22 16:03:33 -08001940 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001941
Jon Hall6aec96b2015-01-19 14:49:31 -08001942 description = "Restore a link to ensure that Link Discovery is " + \
Jon Hall63604932015-02-26 17:09:50 -08001943 "working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001944 main.log.report( description )
1945 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001946
Jon Hall6aec96b2015-01-19 14:49:31 -08001947 main.step( "Bring link between s3 and s28 back up" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001948 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
Jon Hall58c76b72015-02-23 11:09:24 -08001949 main.log.info( "Waiting " + str( linkSleep ) +
1950 " seconds for link up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001951 time.sleep( linkSleep )
1952 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
Jon Hall58c76b72015-02-23 11:09:24 -08001953 onpass="Link up succesful",
1954 onfail="Failed to bring link up" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001955 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -08001956
Jon Hall6aec96b2015-01-19 14:49:31 -08001957 def CASE11( self, main ):
1958 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001959 Switch Down
Jon Hall6aec96b2015-01-19 14:49:31 -08001960 """
1961 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001962 import time
1963
Jon Hall8f89dda2015-01-22 16:03:33 -08001964 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001965
1966 description = "Killing a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001967 main.log.report( description )
1968 main.case( description )
1969 switch = main.params[ 'kill' ][ 'switch' ]
1970 switchDPID = main.params[ 'kill' ][ 'dpid' ]
Jon Hall73cf9cc2014-11-20 22:28:38 -08001971
Jon Hall6aec96b2015-01-19 14:49:31 -08001972 # TODO: Make this switch parameterizable
1973 main.step( "Kill " + switch )
1974 main.log.report( "Deleting " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08001975 main.Mininet1.delSwitch( switch )
1976 main.log.info( "Waiting " + str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001977 " seconds for switch down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001978 time.sleep( switchSleep )
1979 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001980 # Peek at the deleted switch
1981 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001982 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001983 if device and device[ 'available' ] is False:
Jon Hall94fd0472014-12-08 11:52:42 -08001984 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001985 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08001986 onpass="Kill switch succesful",
1987 onfail="Failed to kill switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001988
Jon Hall6aec96b2015-01-19 14:49:31 -08001989 def CASE12( self, main ):
1990 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001991 Switch Up
Jon Hall6aec96b2015-01-19 14:49:31 -08001992 """
1993 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001994 import time
Jon Hall669173b2014-12-17 11:36:30 -08001995
Jon Hall8f89dda2015-01-22 16:03:33 -08001996 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall6aec96b2015-01-19 14:49:31 -08001997 switch = main.params[ 'kill' ][ 'switch' ]
1998 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1999 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hall73cf9cc2014-11-20 22:28:38 -08002000 description = "Adding a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002001 main.log.report( description )
2002 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002003
Jon Hall6aec96b2015-01-19 14:49:31 -08002004 main.step( "Add back " + switch )
2005 main.log.report( "Adding back " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08002006 main.Mininet1.addSwitch( switch, dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002007 # TODO: New dpid or same? Ask Thomas?
2008 for peer in links:
Jon Hall8f89dda2015-01-22 16:03:33 -08002009 main.Mininet1.addLink( switch, peer )
Jon Hall58c76b72015-02-23 11:09:24 -08002010 main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
2011 count=numControllers,
2012 ip1=ONOS1Ip,
2013 port1=ONOS1Port,
2014 ip2=ONOS2Ip,
2015 port2=ONOS2Port,
2016 ip3=ONOS3Ip,
2017 port3=ONOS3Port,
2018 ip4=ONOS4Ip,
2019 port4=ONOS4Port,
2020 ip5=ONOS5Ip,
2021 port5=ONOS5Port,
2022 ip6=ONOS6Ip,
2023 port6=ONOS6Port,
2024 ip7=ONOS7Ip,
2025 port7=ONOS7Port )
2026 main.log.info( "Waiting " + str( switchSleep ) +
2027 " seconds for switch up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002028 time.sleep( switchSleep )
2029 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002030 # Peek at the deleted switch
2031 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08002032 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002033 if device and device[ 'available' ]:
Jon Hall94fd0472014-12-08 11:52:42 -08002034 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002035 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08002036 onpass="add switch succesful",
2037 onfail="Failed to add switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002038
Jon Hall6aec96b2015-01-19 14:49:31 -08002039 def CASE13( self, main ):
2040 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002041 Clean up
Jon Hall6aec96b2015-01-19 14:49:31 -08002042 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002043 import os
2044 import time
Jon Hall6aec96b2015-01-19 14:49:31 -08002045 # TODO: make use of this elsewhere
2046 ips = []
Jon Hall8f89dda2015-01-22 16:03:33 -08002047 ips.append( ONOS1Ip )
2048 ips.append( ONOS2Ip )
2049 ips.append( ONOS3Ip )
2050 ips.append( ONOS4Ip )
2051 ips.append( ONOS5Ip )
2052 ips.append( ONOS6Ip )
2053 ips.append( ONOS7Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -08002054
2055 # printing colors to terminal
Jon Hall669173b2014-12-17 11:36:30 -08002056 colors = {}
Jon Hall6aec96b2015-01-19 14:49:31 -08002057 colors[ 'cyan' ] = '\033[96m'
2058 colors[ 'purple' ] = '\033[95m'
2059 colors[ 'blue' ] = '\033[94m'
2060 colors[ 'green' ] = '\033[92m'
2061 colors[ 'yellow' ] = '\033[93m'
2062 colors[ 'red' ] = '\033[91m'
2063 colors[ 'end' ] = '\033[0m'
Jon Hall73cf9cc2014-11-20 22:28:38 -08002064 description = "Test Cleanup"
Jon Hall6aec96b2015-01-19 14:49:31 -08002065 main.log.report( description )
2066 main.case( description )
2067 main.step( "Killing tcpdumps" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002068 main.Mininet2.stopTcpdump()
Jon Hall73cf9cc2014-11-20 22:28:38 -08002069
Jon Hall6aec96b2015-01-19 14:49:31 -08002070 main.step( "Checking ONOS Logs for errors" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002071 for i in range( 7 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002072 print colors[ 'purple' ] + "Checking logs for errors on " + \
Jon Hall8f89dda2015-01-22 16:03:33 -08002073 "ONOS" + str( i + 1 ) + ":" + colors[ 'end' ]
2074 print main.ONOSbench.checkLogs( ips[ i ] )
Jon Hall94fd0472014-12-08 11:52:42 -08002075
Jon Hall6aec96b2015-01-19 14:49:31 -08002076 main.step( "Copying MN pcap and ONOS log files to test station" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002077 testname = main.TEST
Jon Hall8f89dda2015-01-22 16:03:33 -08002078 teststationUser = main.params[ 'TESTONUSER' ]
2079 teststationIP = main.params[ 'TESTONIP' ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002080 # NOTE: MN Pcap file is being saved to ~/packet_captures
Jon Hall73cf9cc2014-11-20 22:28:38 -08002081 # scp this file as MN and TestON aren't necessarily the same vm
Jon Hall6aec96b2015-01-19 14:49:31 -08002082 # FIXME: scp
2083 # mn files
2084 # TODO: Load these from params
2085 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002086 logFolder = "/opt/onos/log/"
2087 logFiles = [ "karaf.log", "karaf.log.1" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002088 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002089 dstDir = "~/packet_captures/"
2090 for f in logFiles:
2091 for i in range( 7 ):
2092 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
2093 logFolder + f + " " +
2094 teststationUser + "@" +
2095 teststationIP + ":" +
2096 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002097 "-ONOS" + str( i + 1 ) + "-" +
2098 f )
Jon Hall58c76b72015-02-23 11:09:24 -08002099 main.ONOSbench.handle.expect( "\$" )
2100
Jon Hall6aec96b2015-01-19 14:49:31 -08002101 # std*.log's
2102 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002103 logFolder = "/opt/onos/var/"
2104 logFiles = [ "stderr.log", "stdout.log" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002105 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002106 dstDir = "~/packet_captures/"
2107 for f in logFiles:
2108 for i in range( 7 ):
2109 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
2110 logFolder + f + " " +
2111 teststationUser + "@" +
2112 teststationIP + ":" +
2113 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002114 "-ONOS" + str( i + 1 ) + "-" +
2115 f )
Jon Hall58c76b72015-02-23 11:09:24 -08002116 main.ONOSbench.handle.expect( "\$" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002117 # sleep so scp can finish
2118 time.sleep( 10 )
Jon Hall58c76b72015-02-23 11:09:24 -08002119 main.Mininet1.stopNet()
Jon Hall6aec96b2015-01-19 14:49:31 -08002120 main.step( "Packing and rotating pcap archives" )
2121 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002122
Jon Hall6aec96b2015-01-19 14:49:31 -08002123 # TODO: actually check something here
2124 utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08002125 onpass="Test cleanup successful",
2126 onfail="Test cleanup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002127
Jon Hall6aec96b2015-01-19 14:49:31 -08002128 def CASE14( self, main ):
2129 """
Jon Hall669173b2014-12-17 11:36:30 -08002130 start election app on all onos nodes
Jon Hall6aec96b2015-01-19 14:49:31 -08002131 """
Jon Hall8f89dda2015-01-22 16:03:33 -08002132 leaderResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002133 # install app on onos 1
2134 main.log.info( "Install leadership election app" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002135 main.ONOScli1.featureInstall( "onos-app-election" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002136 # wait for election
2137 # check for leader
Jon Hall8f89dda2015-01-22 16:03:33 -08002138 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002139 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08002140 if leader == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08002141 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08002142 pass
Jon Hall6aec96b2015-01-19 14:49:31 -08002143 elif leader is None:
2144 # No leader elected
2145 main.log.report( "No leader was elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002146 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002147 elif leader == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002148 # error in response
2149 # TODO: add check for "Command not found:" in the driver, this
2150 # means the app isn't loaded
Jon Hall8f89dda2015-01-22 16:03:33 -08002151 main.log.report( "Something is wrong with electionTestLeader" +
Jon Hall6aec96b2015-01-19 14:49:31 -08002152 " function, check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002153 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002154 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002155 # error in response
2156 main.log.report(
Jon Hall8f89dda2015-01-22 16:03:33 -08002157 "Unexpected response from electionTestLeader function:'" +
2158 str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002159 "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002160 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002161
Jon Hall6aec96b2015-01-19 14:49:31 -08002162 # install on other nodes and check for leader.
2163 # Should be onos1 and each app should show the same leader
Jon Hall8f89dda2015-01-22 16:03:33 -08002164 for controller in range( 2, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002165 # loop through ONOScli handlers
2166 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002167 node.featureInstall( "onos-app-election" )
2168 leaderN = node.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002169 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08002170 if leaderN == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08002171 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08002172 pass
2173 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002174 # error in response
2175 # TODO: add check for "Command not found:" in the driver, this
2176 # means the app isn't loaded
2177 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002178 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08002179 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002180 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002181 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08002182 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002183 main.log.report( "ONOS" + str( controller ) + " sees " +
2184 str( leaderN ) +
2185 " as the leader of the election app. Leader" +
2186 " should be " +
2187 str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002188 if leaderResult:
Jon Hall6aec96b2015-01-19 14:49:31 -08002189 main.log.report( "Leadership election tests passed( consistent " +
2190 "view of leader across listeners and a leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002191 "was elected )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002192 utilities.assert_equals(
2193 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002194 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002195 onpass="Leadership election passed",
2196 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002197
Jon Hall6aec96b2015-01-19 14:49:31 -08002198 def CASE15( self, main ):
2199 """
Jon Hall669173b2014-12-17 11:36:30 -08002200 Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -08002201 """
Jon Hall8f89dda2015-01-22 16:03:33 -08002202 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002203 description = "Check that Leadership Election is still functional"
Jon Hall6aec96b2015-01-19 14:49:31 -08002204 main.log.report( description )
2205 main.case( description )
2206 main.step( "Find current leader and withdraw" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002207 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002208 # TODO: do some sanity checking on leader before using it
Jon Hall8f89dda2015-01-22 16:03:33 -08002209 withdrawResult = main.FALSE
2210 if leader == ONOS1Ip:
2211 oldLeader = getattr( main, "ONOScli1" )
2212 elif leader == ONOS2Ip:
2213 oldLeader = getattr( main, "ONOScli2" )
2214 elif leader == ONOS3Ip:
2215 oldLeader = getattr( main, "ONOScli3" )
2216 elif leader == ONOS4Ip:
2217 oldLeader = getattr( main, "ONOScli4" )
2218 elif leader == ONOS5Ip:
2219 oldLeader = getattr( main, "ONOScli5" )
2220 elif leader == ONOS6Ip:
2221 oldLeader = getattr( main, "ONOScli6" )
2222 elif leader == ONOS7Ip:
2223 oldLeader = getattr( main, "ONOScli7" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002224 elif leader is None or leader == main.FALSE:
2225 main.log.report(
2226 "Leader for the election app should be an ONOS node," +
Jon Hall58c76b72015-02-23 11:09:24 -08002227 "instead got '" + str( leader ) + "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002228 leaderResult = main.FALSE
Jon Hall63604932015-02-26 17:09:50 -08002229 oldLeader = None
2230 else:
2231 main.log.error( "Leader election --- why am I HERE?!?")
2232 if oldLeader:
2233 withdrawResult = oldLeader.electionTestWithdraw()
Jon Hall6aec96b2015-01-19 14:49:31 -08002234 utilities.assert_equals(
2235 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002236 actual=withdrawResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002237 onpass="App was withdrawn from election",
2238 onfail="App was not withdrawn from election" )
Jon Hall669173b2014-12-17 11:36:30 -08002239
Jon Hall6aec96b2015-01-19 14:49:31 -08002240 main.step( "Make sure new leader is elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002241 leaderList = []
2242 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002243 # loop through ONOScli handlers
2244 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002245 leaderList.append( node.electionTestLeader() )
2246 for leaderN in leaderList:
Jon Hall669173b2014-12-17 11:36:30 -08002247 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08002248 main.log.report(
Jon Hall58c76b72015-02-23 11:09:24 -08002249 "ONOS" + str( controller ) +
2250 " still sees " + str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002251 " as leader after they withdrew" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002252 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002253 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002254 # error in response
2255 # TODO: add check for "Command not found:" in the driver, this
2256 # means the app isn't loaded
2257 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002258 "electionTestLeader function, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002259 "check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002260 leaderResult = main.FALSE
2261 consistentLeader = main.FALSE
2262 if len( set( leaderList ) ) == 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08002263 main.log.info( "Each Election-app sees '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08002264 str( leaderList[ 0 ] ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002265 "' as the leader" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002266 consistentLeader = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002267 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002268 main.log.report(
2269 "Inconsistent responses for leader of Election-app:" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002270 for n in range( len( leaderList ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002271 main.log.report( "ONOS" + str( n + 1 ) + " response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002272 str( leaderList[ n ] ) )
2273 if leaderResult:
2274 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002275 "view of leader across listeners and a new " +
2276 "leader was elected when the old leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002277 "resigned )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002278 utilities.assert_equals(
2279 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002280 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002281 onpass="Leadership election passed",
2282 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002283
Jon Hall58c76b72015-02-23 11:09:24 -08002284 main.step( "Run for election on old leader( just so everyone " +
2285 "is in the hat )" )
Jon Hall63604932015-02-26 17:09:50 -08002286 if oldLeader:
2287 runResult = oldLeader.electionTestRun()
2288 else:
2289 runResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002290 utilities.assert_equals(
2291 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002292 actual=runResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002293 onpass="App re-ran for election",
2294 onfail="App failed to run for election" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002295 if consistentLeader == main.TRUE:
2296 afterRun = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002297 # verify leader didn't just change
Jon Hall8f89dda2015-01-22 16:03:33 -08002298 if afterRun == leaderList[ 0 ]:
2299 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002300 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08002301 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002302 # TODO: assert on run and withdraw results?
Jon Hall669173b2014-12-17 11:36:30 -08002303
Jon Hall6aec96b2015-01-19 14:49:31 -08002304 utilities.assert_equals(
2305 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002306 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002307 onpass="Leadership election passed",
2308 onfail="Something went wrong with Leadership election after " +
2309 "the old leader re-ran for election" )