blob: 9ba287de337c48c51f36b36a29cb9d8a260a58cf [file] [log] [blame]
Jon Hall6aec96b2015-01-19 14:49:31 -08001"""
Jon Hallb1290e82014-11-18 16:17:48 -05002Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case. Since this is the Sanity test, we do nothing.
Jon Hall368769f2014-11-19 15:43:35 -080013CASE7: Check state after control plane failure
Jon Hallb1290e82014-11-18 16:17:48 -050014CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
Jon Hall669173b2014-12-17 11:36:30 -080020CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -080022"""
Jon Hall8f89dda2015-01-22 16:03:33 -080023
24
Jon Hallb1290e82014-11-18 16:17:48 -050025class HATestSanity:
26
Jon Hall6aec96b2015-01-19 14:49:31 -080027 def __init__( self ):
Jon Hallb1290e82014-11-18 16:17:48 -050028 self.default = ''
29
Jon Hall6aec96b2015-01-19 14:49:31 -080030 def CASE1( self, main ):
31 """
Jon Hallb1290e82014-11-18 16:17:48 -050032 CASE1 is to compile ONOS and push it to the test machines
33
34 Startup sequence:
Jon Hallb1290e82014-11-18 16:17:48 -050035 cell <name>
36 onos-verify-cell
37 NOTE: temporary - onos-remove-raft-logs
Jon Hall58c76b72015-02-23 11:09:24 -080038 onos-uninstall
39 start mininet
40 git pull
41 mvn clean install
42 onos-package
Jon Hallb1290e82014-11-18 16:17:48 -050043 onos-install -f
44 onos-wait-for-start
Jon Hall58c76b72015-02-23 11:09:24 -080045 start cli sessions
46 start tcpdump
Jon Hall6aec96b2015-01-19 14:49:31 -080047 """
48 main.log.report( "ONOS HA Sanity test - initialization" )
49 main.case( "Setting up test environment" )
50 # TODO: save all the timers and output them for plotting
Jon Hallb1290e82014-11-18 16:17:48 -050051
52 # load some vairables from the params file
Jon Hall8f89dda2015-01-22 16:03:33 -080053 PULLCODE = False
Jon Hall6aec96b2015-01-19 14:49:31 -080054 if main.params[ 'Git' ] == 'True':
Jon Hall8f89dda2015-01-22 16:03:33 -080055 PULLCODE = True
Jon Hall529a37f2015-01-28 10:02:00 -080056 gitBranch = main.params[ 'branch' ]
Jon Hall8f89dda2015-01-22 16:03:33 -080057 cellName = main.params[ 'ENV' ][ 'cellName' ]
Jon Hall6aec96b2015-01-19 14:49:31 -080058
59 # set global variables
Jon Hall8f89dda2015-01-22 16:03:33 -080060 global ONOS1Ip
61 global ONOS1Port
62 global ONOS2Ip
63 global ONOS2Port
64 global ONOS3Ip
65 global ONOS3Port
66 global ONOS4Ip
67 global ONOS4Port
68 global ONOS5Ip
69 global ONOS5Port
70 global ONOS6Ip
71 global ONOS6Port
72 global ONOS7Ip
73 global ONOS7Port
74 global numControllers
Jon Hallb1290e82014-11-18 16:17:48 -050075
Jon Hall8f89dda2015-01-22 16:03:33 -080076 ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
77 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
78 ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
79 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
80 ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
81 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
82 ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
83 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
84 ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
85 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
86 ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
87 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
88 ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
89 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
90 numControllers = int( main.params[ 'num_controllers' ] )
Jon Hallb1290e82014-11-18 16:17:48 -050091
Jon Hall6aec96b2015-01-19 14:49:31 -080092 main.step( "Applying cell variable to environment" )
Jon Hall8f89dda2015-01-22 16:03:33 -080093 cellResult = main.ONOSbench.setCell( cellName )
94 verifyResult = main.ONOSbench.verifyCell()
Jon Hall368769f2014-11-19 15:43:35 -080095
Jon Hall6aec96b2015-01-19 14:49:31 -080096 # FIXME:this is short term fix
97 main.log.report( "Removing raft logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -080098 main.ONOSbench.onosRemoveRaftLogs()
Jon Hall6aec96b2015-01-19 14:49:31 -080099 main.log.report( "Uninstalling ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800100 main.ONOSbench.onosUninstall( ONOS1Ip )
101 main.ONOSbench.onosUninstall( ONOS2Ip )
102 main.ONOSbench.onosUninstall( ONOS3Ip )
103 main.ONOSbench.onosUninstall( ONOS4Ip )
104 main.ONOSbench.onosUninstall( ONOS5Ip )
105 main.ONOSbench.onosUninstall( ONOS6Ip )
106 main.ONOSbench.onosUninstall( ONOS7Ip )
Jon Hallb1290e82014-11-18 16:17:48 -0500107
Jon Hall8f89dda2015-01-22 16:03:33 -0800108 cleanInstallResult = main.TRUE
109 gitPullResult = main.TRUE
Jon Hallb1290e82014-11-18 16:17:48 -0500110
Jon Hall97f31752015-02-04 12:01:04 -0800111 main.step( "Starting Mininet" )
112 main.Mininet1.startNet( )
113
Jon Hall6aec96b2015-01-19 14:49:31 -0800114 main.step( "Compiling the latest version of ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800115 if PULLCODE:
Jon Hall58c76b72015-02-23 11:09:24 -0800116 main.step( "Git checkout and pull " + gitBranch )
Jon Hall529a37f2015-01-28 10:02:00 -0800117 main.ONOSbench.gitCheckout( gitBranch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800118 gitPullResult = main.ONOSbench.gitPull()
Jon Hallb1290e82014-11-18 16:17:48 -0500119
Jon Hall6aec96b2015-01-19 14:49:31 -0800120 main.step( "Using mvn clean & install" )
Jon Hall529a37f2015-01-28 10:02:00 -0800121 cleanInstallResult = main.ONOSbench.cleanInstall()
122 else:
123 main.log.warn( "Did not pull new code so skipping mvn " +
124 "clean install" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800125 main.ONOSbench.getVersion( report=True )
Jon Hallb1290e82014-11-18 16:17:48 -0500126
Jon Hall6aec96b2015-01-19 14:49:31 -0800127 main.step( "Creating ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800128 packageResult = main.ONOSbench.onosPackage()
Jon Hallb1290e82014-11-18 16:17:48 -0500129
Jon Hall6aec96b2015-01-19 14:49:31 -0800130 main.step( "Installing ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800131 onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
132 node=ONOS1Ip )
133 onos2InstallResult = main.ONOSbench.onosInstall( options="-f",
134 node=ONOS2Ip )
135 onos3InstallResult = main.ONOSbench.onosInstall( options="-f",
136 node=ONOS3Ip )
137 onos4InstallResult = main.ONOSbench.onosInstall( options="-f",
138 node=ONOS4Ip )
139 onos5InstallResult = main.ONOSbench.onosInstall( options="-f",
140 node=ONOS5Ip )
141 onos6InstallResult = main.ONOSbench.onosInstall( options="-f",
142 node=ONOS6Ip )
143 onos7InstallResult = main.ONOSbench.onosInstall( options="-f",
144 node=ONOS7Ip )
145 onosInstallResult = onos1InstallResult and onos2InstallResult\
146 and onos3InstallResult and onos4InstallResult\
147 and onos5InstallResult and onos6InstallResult\
148 and onos7InstallResult
Jon Hallb1290e82014-11-18 16:17:48 -0500149
Jon Hall6aec96b2015-01-19 14:49:31 -0800150 main.step( "Checking if ONOS is up yet" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800151 for i in range( 2 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800152 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
153 if not onos1Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800154 main.log.report( "ONOS1 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800155 main.ONOSbench.onosStop( ONOS1Ip )
156 main.ONOSbench.onosStart( ONOS1Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800157 onos2Isup = main.ONOSbench.isup( ONOS2Ip )
158 if not onos2Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800159 main.log.report( "ONOS2 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800160 main.ONOSbench.onosStop( ONOS2Ip )
161 main.ONOSbench.onosStart( ONOS2Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800162 onos3Isup = main.ONOSbench.isup( ONOS3Ip )
163 if not onos3Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800164 main.log.report( "ONOS3 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800165 main.ONOSbench.onosStop( ONOS3Ip )
166 main.ONOSbench.onosStart( ONOS3Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800167 onos4Isup = main.ONOSbench.isup( ONOS4Ip )
168 if not onos4Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800169 main.log.report( "ONOS4 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800170 main.ONOSbench.onosStop( ONOS4Ip )
171 main.ONOSbench.onosStart( ONOS4Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800172 onos5Isup = main.ONOSbench.isup( ONOS5Ip )
173 if not onos5Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800174 main.log.report( "ONOS5 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800175 main.ONOSbench.onosStop( ONOS5Ip )
176 main.ONOSbench.onosStart( ONOS5Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800177 onos6Isup = main.ONOSbench.isup( ONOS6Ip )
178 if not onos6Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800179 main.log.report( "ONOS6 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800180 main.ONOSbench.onosStop( ONOS6Ip )
181 main.ONOSbench.onosStart( ONOS6Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800182 onos7Isup = main.ONOSbench.isup( ONOS7Ip )
183 if not onos7Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800184 main.log.report( "ONOS7 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800185 main.ONOSbench.onosStop( ONOS7Ip )
186 main.ONOSbench.onosStart( ONOS7Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800187 onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
188 and onos4Isup and onos5Isup and onos6Isup and onos7Isup
189 if onosIsupResult == main.TRUE:
Jon Hallffb386d2014-11-21 13:43:38 -0800190 break
Jon Hallb1290e82014-11-18 16:17:48 -0500191
Jon Hall8f89dda2015-01-22 16:03:33 -0800192 cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
193 cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
194 cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
195 cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
196 cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
197 cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
198 cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
199 cliResults = cliResult1 and cliResult2 and cliResult3 and\
200 cliResult4 and cliResult5 and cliResult6 and cliResult7
Jon Hallb1290e82014-11-18 16:17:48 -0500201
Jon Hall6aec96b2015-01-19 14:49:31 -0800202 main.step( "Start Packet Capture MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800203 main.Mininet2.startTcpdump(
Jon Hall6aec96b2015-01-19 14:49:31 -0800204 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
205 + "-MN.pcap",
206 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
207 port=main.params[ 'MNtcpdump' ][ 'port' ] )
Jon Hallb1290e82014-11-18 16:17:48 -0500208
Jon Hall8f89dda2015-01-22 16:03:33 -0800209 case1Result = ( cleanInstallResult and packageResult and
210 cellResult and verifyResult and onosInstallResult
211 and onosIsupResult and cliResults )
Jon Hallb1290e82014-11-18 16:17:48 -0500212
Jon Hall8f89dda2015-01-22 16:03:33 -0800213 utilities.assert_equals( expect=main.TRUE, actual=case1Result,
Jon Hall58c76b72015-02-23 11:09:24 -0800214 onpass="Test startup successful",
215 onfail="Test startup NOT successful" )
Jon Hallb1290e82014-11-18 16:17:48 -0500216
Jon Hall8f89dda2015-01-22 16:03:33 -0800217 if case1Result == main.FALSE:
Jon Hallffb386d2014-11-21 13:43:38 -0800218 main.cleanup()
219 main.exit()
Jon Hallb1290e82014-11-18 16:17:48 -0500220
Jon Hall6aec96b2015-01-19 14:49:31 -0800221 def CASE2( self, main ):
222 """
Jon Hallb1290e82014-11-18 16:17:48 -0500223 Assign mastership to controllers
Jon Hall6aec96b2015-01-19 14:49:31 -0800224 """
Jon Hallb1290e82014-11-18 16:17:48 -0500225 import re
226
Jon Hall6aec96b2015-01-19 14:49:31 -0800227 main.log.report( "Assigning switches to controllers" )
228 main.case( "Assigning Controllers" )
229 main.step( "Assign switches to controllers" )
Jon Hallb1290e82014-11-18 16:17:48 -0500230
Jon Hall6aec96b2015-01-19 14:49:31 -0800231 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800232 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -0800233 sw=str( i ),
Jon Hall8f89dda2015-01-22 16:03:33 -0800234 count=numControllers,
235 ip1=ONOS1Ip, port1=ONOS1Port,
236 ip2=ONOS2Ip, port2=ONOS2Port,
237 ip3=ONOS3Ip, port3=ONOS3Port,
238 ip4=ONOS4Ip, port4=ONOS4Port,
239 ip5=ONOS5Ip, port5=ONOS5Port,
240 ip6=ONOS6Ip, port6=ONOS6Port,
241 ip7=ONOS7Ip, port7=ONOS7Port )
Jon Hallb1290e82014-11-18 16:17:48 -0500242
Jon Hall8f89dda2015-01-22 16:03:33 -0800243 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800244 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800245 response = main.Mininet1.getSwController( "s" + str( i ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800246 try:
Jon Hall6aec96b2015-01-19 14:49:31 -0800247 main.log.info( str( response ) )
Jon Hallfebb1c72015-03-05 13:30:09 -0800248 except Exception:
Jon Hall6aec96b2015-01-19 14:49:31 -0800249 main.log.info( repr( response ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800250 if re.search( "tcp:" + ONOS1Ip, response )\
251 and re.search( "tcp:" + ONOS2Ip, response )\
252 and re.search( "tcp:" + ONOS3Ip, response )\
253 and re.search( "tcp:" + ONOS4Ip, response )\
254 and re.search( "tcp:" + ONOS5Ip, response )\
255 and re.search( "tcp:" + ONOS6Ip, response )\
256 and re.search( "tcp:" + ONOS7Ip, response ):
257 mastershipCheck = mastershipCheck and main.TRUE
Jon Hallb1290e82014-11-18 16:17:48 -0500258 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800259 mastershipCheck = main.FALSE
260 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800261 main.log.report( "Switch mastership assigned correctly" )
262 utilities.assert_equals(
263 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800264 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800265 onpass="Switch mastership assigned correctly",
266 onfail="Switches not assigned correctly to controllers" )
Jon Hallb1290e82014-11-18 16:17:48 -0500267
Jon Hall6aec96b2015-01-19 14:49:31 -0800268 # Manually assign mastership to the controller we want
Jon Hall8f89dda2015-01-22 16:03:33 -0800269 roleCall = main.TRUE
270 roleCheck = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -0800271 try:
272 # Assign switch
273 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
274 assert deviceId, "No device id for s1 in ONOS"
Jon Hall8f89dda2015-01-22 16:03:33 -0800275 roleCall = roleCall and main.ONOScli1.deviceRole(
276 deviceId,
Jon Hall58c76b72015-02-23 11:09:24 -0800277 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800278 # Check assignment
Jon Hall58c76b72015-02-23 11:09:24 -0800279 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800280 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800281 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800282 roleCheck = roleCheck and main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800283
Jon Hall58c76b72015-02-23 11:09:24 -0800284 # Assign switch
285 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
286 assert deviceId, "No device id for s28 in ONOS"
Jon Hall8f89dda2015-01-22 16:03:33 -0800287 roleCall = roleCall and main.ONOScli1.deviceRole(
288 deviceId,
Jon Hall58c76b72015-02-23 11:09:24 -0800289 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800290 # Check assignment
Jon Hall58c76b72015-02-23 11:09:24 -0800291 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800292 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800293 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800294 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800295
Jon Hall58c76b72015-02-23 11:09:24 -0800296 # Assign switch
297 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
298 assert deviceId, "No device id for s2 in ONOS"
299 roleCall = roleCall and main.ONOScli1.deviceRole(
300 deviceId,
301 ONOS2Ip )
302 # Check assignment
303 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
304 roleCheck = roleCheck and main.TRUE
305 else:
306 roleCheck = roleCheck and main.FALSE
307
308 # Assign switch
309 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
310 assert deviceId, "No device id for s3 in ONOS"
311 roleCall = roleCall and main.ONOScli1.deviceRole(
312 deviceId,
313 ONOS2Ip )
314 # Check assignment
315 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
316 roleCheck = roleCheck and main.TRUE
317 else:
318 roleCheck = roleCheck and main.FALSE
319
320 # Assign switch
321 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
322 assert deviceId, "No device id for s5 in ONOS"
323 roleCall = roleCall and main.ONOScli1.deviceRole(
324 deviceId,
325 ONOS3Ip )
326 # Check assignment
327 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
328 roleCheck = roleCheck and main.TRUE
329 else:
330 roleCheck = roleCheck and main.FALSE
331
332 # Assign switch
333 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
334 assert deviceId, "No device id for s6 in ONOS"
335 roleCall = roleCall and main.ONOScli1.deviceRole(
336 deviceId,
337 ONOS3Ip )
338 # Check assignment
339 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
340 roleCheck = roleCheck and main.TRUE
341 else:
342 roleCheck = roleCheck and main.FALSE
343
344 # Assign switch
345 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
346 assert deviceId, "No device id for s4 in ONOS"
347 roleCall = roleCall and main.ONOScli1.deviceRole(
348 deviceId,
349 ONOS4Ip )
350 # Check assignment
351 if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
352 roleCheck = roleCheck and main.TRUE
353 else:
354 roleCheck = roleCheck and main.FALSE
355
356 for i in range( 8, 18 ):
357 dpid = '3' + str( i ).zfill( 3 )
358 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
359 assert deviceId, "No device id for s%i in ONOS" % i
360 roleCall = roleCall and main.ONOScli1.deviceRole(
361 deviceId,
362 ONOS5Ip )
363 # Check assignment
364 if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
365 roleCheck = roleCheck and main.TRUE
366 else:
367 roleCheck = roleCheck and main.FALSE
368
369 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
370 assert deviceId, "No device id for s7 in ONOS"
371 roleCall = roleCall and main.ONOScli1.deviceRole(
372 deviceId,
373 ONOS6Ip )
374 # Check assignment
375 if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
376 roleCheck = roleCheck and main.TRUE
377 else:
378 roleCheck = roleCheck and main.FALSE
379
380 for i in range( 18, 28 ):
381 dpid = '6' + str( i ).zfill( 3 )
382 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
383 assert deviceId, "No device id for s%i in ONOS" % i
384 roleCall = roleCall and main.ONOScli1.deviceRole(
385 deviceId,
386 ONOS7Ip )
387 # Check assignment
388 if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
389 roleCheck = roleCheck and main.TRUE
390 else:
391 roleCheck = roleCheck and main.FALSE
392 except ( AttributeError, AssertionError ):
393 main.log.exception( "Something is wrong with ONOS device view" )
394 main.log.info( main.ONOScli1.devices() )
395
Jon Hall6aec96b2015-01-19 14:49:31 -0800396 utilities.assert_equals(
397 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800398 actual=roleCall,
Jon Hall6aec96b2015-01-19 14:49:31 -0800399 onpass="Re-assigned switch mastership to designated controller",
Jon Hall8f89dda2015-01-22 16:03:33 -0800400 onfail="Something wrong with deviceRole calls" )
Jon Hall94fd0472014-12-08 11:52:42 -0800401
Jon Hall6aec96b2015-01-19 14:49:31 -0800402 utilities.assert_equals(
403 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800404 actual=roleCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800405 onpass="Switches were successfully reassigned to designated " +
406 "controller",
407 onfail="Switches were not successfully reassigned" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800408 mastershipCheck = mastershipCheck and roleCall and roleCheck
409 utilities.assert_equals( expect=main.TRUE, actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800410 onpass="Switch mastership correctly assigned",
411 onfail="Error in (re)assigning switch" +
Jon Hall8f89dda2015-01-22 16:03:33 -0800412 " mastership" )
Jon Hallb1290e82014-11-18 16:17:48 -0500413
Jon Hall6aec96b2015-01-19 14:49:31 -0800414 def CASE3( self, main ):
Jon Hallb1290e82014-11-18 16:17:48 -0500415 """
416 Assign intents
Jon Hallb1290e82014-11-18 16:17:48 -0500417 """
418 import time
Jon Hall58c76b72015-02-23 11:09:24 -0800419 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800420 main.log.report( "Adding host intents" )
421 main.case( "Adding host Intents" )
Jon Hallb1290e82014-11-18 16:17:48 -0500422
Jon Hall8f89dda2015-01-22 16:03:33 -0800423 main.step( "Discovering Hosts( Via pingall for now )" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800424 # FIXME: Once we have a host discovery mechanism, use that instead
Jon Hallb1290e82014-11-18 16:17:48 -0500425
Jon Hall6aec96b2015-01-19 14:49:31 -0800426 # install onos-app-fwd
427 main.log.info( "Install reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800428 main.ONOScli1.featureInstall( "onos-app-fwd" )
429 main.ONOScli2.featureInstall( "onos-app-fwd" )
430 main.ONOScli3.featureInstall( "onos-app-fwd" )
431 main.ONOScli4.featureInstall( "onos-app-fwd" )
432 main.ONOScli5.featureInstall( "onos-app-fwd" )
433 main.ONOScli6.featureInstall( "onos-app-fwd" )
434 main.ONOScli7.featureInstall( "onos-app-fwd" )
Jon Hall94fd0472014-12-08 11:52:42 -0800435
Jon Hall6aec96b2015-01-19 14:49:31 -0800436 # REACTIVE FWD test
Jon Hall8f89dda2015-01-22 16:03:33 -0800437 pingResult = main.FALSE
Jon Hallb1290e82014-11-18 16:17:48 -0500438 time1 = time.time()
Jon Hall8f89dda2015-01-22 16:03:33 -0800439 pingResult = main.Mininet1.pingall()
Jon Hall529a37f2015-01-28 10:02:00 -0800440 utilities.assert_equals(
441 expect=main.TRUE,
442 actual=pingResult,
443 onpass="Reactive Pingall test passed",
444 onfail="Reactive Pingall failed, one or more ping pairs failed" )
Jon Hallb1290e82014-11-18 16:17:48 -0500445 time2 = time.time()
Jon Hall6aec96b2015-01-19 14:49:31 -0800446 main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
Jon Hallb1290e82014-11-18 16:17:48 -0500447
Jon Hall6aec96b2015-01-19 14:49:31 -0800448 # uninstall onos-app-fwd
449 main.log.info( "Uninstall reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800450 main.ONOScli1.featureUninstall( "onos-app-fwd" )
451 main.ONOScli2.featureUninstall( "onos-app-fwd" )
452 main.ONOScli3.featureUninstall( "onos-app-fwd" )
453 main.ONOScli4.featureUninstall( "onos-app-fwd" )
454 main.ONOScli5.featureUninstall( "onos-app-fwd" )
455 main.ONOScli6.featureUninstall( "onos-app-fwd" )
456 main.ONOScli7.featureUninstall( "onos-app-fwd" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800457 # timeout for fwd flows
458 time.sleep( 10 )
Jon Hallb1290e82014-11-18 16:17:48 -0500459
Jon Hall6aec96b2015-01-19 14:49:31 -0800460 main.step( "Add host intents" )
Jon Hall58c76b72015-02-23 11:09:24 -0800461 intentIds = []
Jon Hall6aec96b2015-01-19 14:49:31 -0800462 # TODO: move the host numbers to params
Jon Hall58c76b72015-02-23 11:09:24 -0800463 # Maybe look at all the paths we ping?
Jon Hall8f89dda2015-01-22 16:03:33 -0800464 intentAddResult = True
Jon Hall58c76b72015-02-23 11:09:24 -0800465 hostResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800466 for i in range( 8, 18 ):
467 main.log.info( "Adding host intent between h" + str( i ) +
468 " and h" + str( i + 10 ) )
469 host1 = "00:00:00:00:00:" + \
470 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
471 host2 = "00:00:00:00:00:" + \
472 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800473 # NOTE: getHost can return None
474 host1Dict = main.ONOScli1.getHost( host1 )
475 host2Dict = main.ONOScli1.getHost( host2 )
476 host1Id = None
477 host2Id = None
478 if host1Dict and host2Dict:
479 host1Id = host1Dict.get( 'id', None )
480 host2Id = host2Dict.get( 'id', None )
Jon Hall8f89dda2015-01-22 16:03:33 -0800481 if host1Id and host2Id:
Jon Hall58c76b72015-02-23 11:09:24 -0800482 nodeNum = ( i % 7 ) + 1
483 node = getattr( main, ( 'ONOScli' + str( nodeNum ) ) )
484 tmpId = node.addHostIntent(
Jon Hall8f89dda2015-01-22 16:03:33 -0800485 host1Id,
486 host2Id )
Jon Hall63604932015-02-26 17:09:50 -0800487 if tmpId:
488 main.log.info( "Added intent with id: " + tmpId )
489 intentIds.append( tmpId )
490 else:
491 main.log.error( "addHostIntent reutrned None" )
Jon Hall669173b2014-12-17 11:36:30 -0800492 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800493 main.log.error( "Error, getHost() failed" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800494 main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
495 sort_keys=True,
496 indent=4,
497 separators=( ',', ': ' ) ) )
Jon Hall58c76b72015-02-23 11:09:24 -0800498 hostResult = main.FALSE
499 onosIds = main.ONOScli1.getAllIntentsId()
500 main.log.info( "Submitted intents: " + str( intentIds ) )
501 main.log.info( "Intents in ONOS: " + str( onosIds ) )
502 for intent in intentIds:
503 if intent in onosIds:
504 pass # intent submitted is still in onos
505 else:
506 intentAddResult = False
Jon Hall1b8f54a2015-02-04 13:24:20 -0800507 # Print the intent states
Jon Hall58c76b72015-02-23 11:09:24 -0800508 intents = main.ONOScli1.intents()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800509 intentStates = []
Jon Hall63604932015-02-26 17:09:50 -0800510 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800511 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
512 count = 0
Jon Hall1b8f54a2015-02-04 13:24:20 -0800513 for intent in json.loads( intents ): # Iter through intents of a node
Jon Hall58c76b72015-02-23 11:09:24 -0800514 state = intent.get( 'state', None )
Jon Hall63604932015-02-26 17:09:50 -0800515 if "INSTALLED" not in state:
516 installedCheck = False
Jon Hall58c76b72015-02-23 11:09:24 -0800517 intentId = intent.get( 'id', None )
518 intentStates.append( ( intentId, state ) )
519 # add submitted intents not in the store
520 tmplist = [ i for i, s in intentStates ]
521 missingIntents = False
522 for i in intentIds:
523 if i not in tmplist:
524 intentStates.append( ( i, " - " ) )
525 missingIntents = True
526 intentStates.sort()
527 for i, s in intentStates:
528 count += 1
529 main.log.info( "%-6s%-15s%-15s" %
530 ( str( count ), str( i ), str( s ) ) )
Jon Hall63604932015-02-26 17:09:50 -0800531 main.ONOScli1.leaders()
532 main.ONOScli1.partitions()
533 # for node in nodes:
534 # node.pendingMap()
535 pendingMap = main.ONOScli1.pendingMap()
536 main.ONOScli2.pendingMap()
537 main.ONOScli3.pendingMap()
538 main.ONOScli4.pendingMap()
539 main.ONOScli5.pendingMap()
540 main.ONOScli6.pendingMap()
541 main.ONOScli7.pendingMap()
Jon Hall58c76b72015-02-23 11:09:24 -0800542 intentAddResult = bool( pingResult and hostResult and intentAddResult
Jon Hall63604932015-02-26 17:09:50 -0800543 and not missingIntents and installedCheck )
Jon Hall6aec96b2015-01-19 14:49:31 -0800544 utilities.assert_equals(
545 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800546 actual=intentAddResult,
547 onpass="Pushed host intents to ONOS",
548 onfail="Error in pushing host intents to ONOS" )
Jon Hall58c76b72015-02-23 11:09:24 -0800549
Jon Hall63604932015-02-26 17:09:50 -0800550 if not intentAddResult or "key" in pendingMap:
Jon Hall58c76b72015-02-23 11:09:24 -0800551 import time
Jon Hall63604932015-02-26 17:09:50 -0800552 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800553 main.log.info( "Sleeping 60 seconds to see if intents are found" )
554 time.sleep( 60 )
555 onosIds = main.ONOScli1.getAllIntentsId()
556 main.log.info( "Submitted intents: " + str( intentIds ) )
557 main.log.info( "Intents in ONOS: " + str( onosIds ) )
558 # Print the intent states
559 intents = main.ONOScli1.intents()
560 intentStates = []
561 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
562 count = 0
563 for intent in json.loads( intents ):
564 # Iter through intents of a node
565 state = intent.get( 'state', None )
Jon Hall63604932015-02-26 17:09:50 -0800566 if "INSTALLED" not in state:
567 installedCheck = False
Jon Hall58c76b72015-02-23 11:09:24 -0800568 intentId = intent.get( 'id', None )
569 intentStates.append( ( intentId, state ) )
570 # add submitted intents not in the store
571 tmplist = [ i for i, s in intentStates ]
572 for i in intentIds:
573 if i not in tmplist:
574 intentStates.append( ( i, " - " ) )
575 intentStates.sort()
576 for i, s in intentStates:
577 count += 1
578 main.log.info( "%-6s%-15s%-15s" %
579 ( str( count ), str( i ), str( s ) ) )
Jon Hall63604932015-02-26 17:09:50 -0800580 main.ONOScli1.leaders()
581 main.ONOScli1.pendingMap()
582 main.ONOScli2.pendingMap()
583 main.ONOScli3.pendingMap()
584 main.ONOScli4.pendingMap()
585 main.ONOScli5.pendingMap()
586 main.ONOScli6.pendingMap()
587 main.ONOScli7.pendingMap()
Jon Hallb1290e82014-11-18 16:17:48 -0500588
Jon Hall6aec96b2015-01-19 14:49:31 -0800589 def CASE4( self, main ):
Jon Hallb1290e82014-11-18 16:17:48 -0500590 """
591 Ping across added host intents
592 """
Jon Hall58c76b72015-02-23 11:09:24 -0800593 import json
Jon Hall368769f2014-11-19 15:43:35 -0800594 description = " Ping across added host intents"
Jon Hall6aec96b2015-01-19 14:49:31 -0800595 main.log.report( description )
596 main.case( description )
Jon Hall8f89dda2015-01-22 16:03:33 -0800597 PingResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800598 for i in range( 8, 18 ):
Jon Hall58c76b72015-02-23 11:09:24 -0800599 ping = main.Mininet1.pingHost( src="h" + str( i ),
600 target="h" + str( i + 10 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800601 PingResult = PingResult and ping
Jon Hall6aec96b2015-01-19 14:49:31 -0800602 if ping == main.FALSE:
603 main.log.warn( "Ping failed between h" + str( i ) +
604 " and h" + str( i + 10 ) )
605 elif ping == main.TRUE:
606 main.log.info( "Ping test passed!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800607 # Don't set PingResult or you'd override failures
Jon Hall8f89dda2015-01-22 16:03:33 -0800608 if PingResult == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800609 main.log.report(
610 "Intents have not been installed correctly, pings failed." )
Jon Hall58c76b72015-02-23 11:09:24 -0800611 # TODO: pretty print
Jon Hall529a37f2015-01-28 10:02:00 -0800612 main.log.warn( "ONSO1 intents: " )
613 main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
614 sort_keys=True,
615 indent=4,
616 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800617 if PingResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800618 main.log.report(
619 "Intents have been installed correctly and verified by pings" )
620 utilities.assert_equals(
621 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800622 actual=PingResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800623 onpass="Intents have been installed correctly and pings work",
624 onfail="Intents have not been installed correctly, pings failed." )
Jon Hallb1290e82014-11-18 16:17:48 -0500625
Jon Hall63604932015-02-26 17:09:50 -0800626 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800627 if PingResult is not main.TRUE:
628 # Print the intent states
629 intents = main.ONOScli1.intents()
630 intentStates = []
631 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
632 count = 0
633 # Iter through intents of a node
634 for intent in json.loads( intents ):
635 state = intent.get( 'state', None )
Jon Hall63604932015-02-26 17:09:50 -0800636 if "INSTALLED" not in state:
637 installedCheck = False
Jon Hall58c76b72015-02-23 11:09:24 -0800638 intentId = intent.get( 'id', None )
639 intentStates.append( ( intentId, state ) )
640 intentStates.sort()
641 for i, s in intentStates:
642 count += 1
643 main.log.info( "%-6s%-15s%-15s" %
644 ( str( count ), str( i ), str( s ) ) )
Jon Hall63604932015-02-26 17:09:50 -0800645 main.ONOScli1.leaders()
646 main.ONOScli1.partitions()
647 main.ONOScli1.pendingMap()
648 main.ONOScli2.pendingMap()
649 main.ONOScli3.pendingMap()
650 main.ONOScli4.pendingMap()
651 main.ONOScli5.pendingMap()
652 main.ONOScli6.pendingMap()
653 main.ONOScli7.pendingMap()
654 if not installedCheck:
655 main.log.info( "Waiting 60 seconds to see if intent states change" )
656 time.sleep( 60 )
657 # Print the intent states
658 intents = main.ONOScli1.intents()
659 intentStates = []
660 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
661 count = 0
662 # Iter through intents of a node
663 for intent in json.loads( intents ):
664 state = intent.get( 'state', None )
665 if "INSTALLED" not in state:
666 installedCheck = False
667 intentId = intent.get( 'id', None )
668 intentStates.append( ( intentId, state ) )
669 intentStates.sort()
670 for i, s in intentStates:
671 count += 1
672 main.log.info( "%-6s%-15s%-15s" %
673 ( str( count ), str( i ), str( s ) ) )
674 main.ONOScli1.leaders()
675 main.ONOScli1.partitions()
676 main.ONOScli1.pendingMap()
677 main.ONOScli2.pendingMap()
678 main.ONOScli3.pendingMap()
679 main.ONOScli4.pendingMap()
680 main.ONOScli5.pendingMap()
681 main.ONOScli6.pendingMap()
682 main.ONOScli7.pendingMap()
Jon Hallb1290e82014-11-18 16:17:48 -0500683
Jon Hall6aec96b2015-01-19 14:49:31 -0800684 def CASE5( self, main ):
685 """
Jon Hallb1290e82014-11-18 16:17:48 -0500686 Reading state of ONOS
Jon Hall6aec96b2015-01-19 14:49:31 -0800687 """
Jon Hallb1290e82014-11-18 16:17:48 -0500688 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800689 # assumes that sts is already in you PYTHONPATH
690 from sts.topology.teston_topology import TestONTopology
Jon Hallb1290e82014-11-18 16:17:48 -0500691
Jon Hall6aec96b2015-01-19 14:49:31 -0800692 main.log.report( "Setting up and gathering data for current state" )
693 main.case( "Setting up and gathering data for current state" )
694 # The general idea for this test case is to pull the state of
695 # ( intents,flows, topology,... ) from each ONOS node
696 # We can then compare them with eachother and also with past states
Jon Hallb1290e82014-11-18 16:17:48 -0500697
Jon Hall6aec96b2015-01-19 14:49:31 -0800698 main.step( "Get the Mastership of each switch from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800699 global mastershipState
700 mastershipState = []
Jon Hall94fd0472014-12-08 11:52:42 -0800701
Jon Hall6aec96b2015-01-19 14:49:31 -0800702 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -0800703 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
704 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
705 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
706 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
707 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
708 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
709 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
710 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
711 ONOS3MasterNotNull and ONOS4MasterNotNull and\
712 ONOS5MasterNotNull and ONOS6MasterNotNull and\
713 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -0800714 utilities.assert_equals(
715 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800716 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -0800717 onpass="Each device has a master",
718 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -0800719
Jon Hall8f89dda2015-01-22 16:03:33 -0800720 ONOS1Mastership = main.ONOScli1.roles()
721 ONOS2Mastership = main.ONOScli2.roles()
722 ONOS3Mastership = main.ONOScli3.roles()
723 ONOS4Mastership = main.ONOScli4.roles()
724 ONOS5Mastership = main.ONOScli5.roles()
725 ONOS6Mastership = main.ONOScli6.roles()
726 ONOS7Mastership = main.ONOScli7.roles()
Jon Hall8f89dda2015-01-22 16:03:33 -0800727 if "Error" in ONOS1Mastership or not ONOS1Mastership\
728 or "Error" in ONOS2Mastership or not ONOS2Mastership\
729 or "Error" in ONOS3Mastership or not ONOS3Mastership\
730 or "Error" in ONOS4Mastership or not ONOS4Mastership\
731 or "Error" in ONOS5Mastership or not ONOS5Mastership\
732 or "Error" in ONOS6Mastership or not ONOS6Mastership\
733 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -0800734 main.log.report( "Error in getting ONOS roles" )
735 main.log.warn(
736 "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800737 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800738 main.log.warn(
739 "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800740 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800741 main.log.warn(
742 "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800743 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800744 main.log.warn(
745 "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800746 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800747 main.log.warn(
748 "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800749 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800750 main.log.warn(
751 "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800752 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800753 main.log.warn(
754 "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800755 repr( ONOS7Mastership ) )
756 consistentMastership = main.FALSE
757 elif ONOS1Mastership == ONOS2Mastership\
758 and ONOS1Mastership == ONOS3Mastership\
759 and ONOS1Mastership == ONOS4Mastership\
760 and ONOS1Mastership == ONOS5Mastership\
761 and ONOS1Mastership == ONOS6Mastership\
762 and ONOS1Mastership == ONOS7Mastership:
763 mastershipState = ONOS1Mastership
764 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800765 main.log.report(
766 "Switch roles are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -0500767 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800768 main.log.warn(
769 "ONOS1 roles: ",
770 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800771 json.loads( ONOS1Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800772 sort_keys=True,
773 indent=4,
774 separators=(
775 ',',
776 ': ' ) ) )
777 main.log.warn(
778 "ONOS2 roles: ",
779 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800780 json.loads( ONOS2Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800781 sort_keys=True,
782 indent=4,
783 separators=(
784 ',',
785 ': ' ) ) )
786 main.log.warn(
787 "ONOS3 roles: ",
788 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800789 json.loads( ONOS3Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800790 sort_keys=True,
791 indent=4,
792 separators=(
793 ',',
794 ': ' ) ) )
795 main.log.warn(
796 "ONOS4 roles: ",
797 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800798 json.loads( ONOS4Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800799 sort_keys=True,
800 indent=4,
801 separators=(
802 ',',
803 ': ' ) ) )
804 main.log.warn(
805 "ONOS5 roles: ",
806 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800807 json.loads( ONOS5Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800808 sort_keys=True,
809 indent=4,
810 separators=(
811 ',',
812 ': ' ) ) )
813 main.log.warn(
814 "ONOS6 roles: ",
815 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800816 json.loads( ONOS6Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800817 sort_keys=True,
818 indent=4,
819 separators=(
820 ',',
821 ': ' ) ) )
822 main.log.warn(
823 "ONOS7 roles: ",
824 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800825 json.loads( ONOS7Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800826 sort_keys=True,
827 indent=4,
828 separators=(
829 ',',
830 ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800831 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800832 utilities.assert_equals(
833 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800834 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -0800835 onpass="Switch roles are consistent across all ONOS nodes",
836 onfail="ONOS nodes have different views of switch roles" )
Jon Hallb1290e82014-11-18 16:17:48 -0500837
Jon Hall6aec96b2015-01-19 14:49:31 -0800838 main.step( "Get the intents from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800839 global intentState
840 intentState = []
841 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
842 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
843 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
844 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
845 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
846 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
847 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
848 intentCheck = main.FALSE
849 if "Error" in ONOS1Intents or not ONOS1Intents\
850 or "Error" in ONOS2Intents or not ONOS2Intents\
851 or "Error" in ONOS3Intents or not ONOS3Intents\
852 or "Error" in ONOS4Intents or not ONOS4Intents\
853 or "Error" in ONOS5Intents or not ONOS5Intents\
854 or "Error" in ONOS6Intents or not ONOS6Intents\
855 or "Error" in ONOS7Intents or not ONOS7Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -0800856 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800857 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
858 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
859 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
860 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
861 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
862 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
863 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
864 elif ONOS1Intents == ONOS2Intents\
865 and ONOS1Intents == ONOS3Intents\
866 and ONOS1Intents == ONOS4Intents\
867 and ONOS1Intents == ONOS5Intents\
868 and ONOS1Intents == ONOS6Intents\
869 and ONOS1Intents == ONOS7Intents:
870 intentState = ONOS1Intents
871 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800872 main.log.report( "Intents are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -0500873 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800874 main.log.warn(
875 "ONOS1 intents: ",
876 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800877 json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800878 sort_keys=True,
879 indent=4,
880 separators=(
881 ',',
882 ': ' ) ) )
883 main.log.warn(
884 "ONOS2 intents: ",
885 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800886 json.loads( ONOS2Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800887 sort_keys=True,
888 indent=4,
889 separators=(
890 ',',
891 ': ' ) ) )
892 main.log.warn(
893 "ONOS3 intents: ",
894 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800895 json.loads( ONOS3Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800896 sort_keys=True,
897 indent=4,
898 separators=(
899 ',',
900 ': ' ) ) )
901 main.log.warn(
902 "ONOS4 intents: ",
903 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800904 json.loads( ONOS4Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800905 sort_keys=True,
906 indent=4,
907 separators=(
908 ',',
909 ': ' ) ) )
910 main.log.warn(
911 "ONOS5 intents: ",
912 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800913 json.loads( ONOS5Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800914 sort_keys=True,
915 indent=4,
916 separators=(
917 ',',
918 ': ' ) ) )
919 main.log.warn(
920 "ONOS6 intents: ",
921 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800922 json.loads( ONOS6Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800923 sort_keys=True,
924 indent=4,
925 separators=(
926 ',',
927 ': ' ) ) )
928 main.log.warn(
929 "ONOS7 intents: ",
930 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800931 json.loads( ONOS7Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800932 sort_keys=True,
933 indent=4,
934 separators=(
935 ',',
936 ': ' ) ) )
937 utilities.assert_equals(
938 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800939 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800940 onpass="Intents are consistent across all ONOS nodes",
941 onfail="ONOS nodes have different views of intents" )
Jon Hallb1290e82014-11-18 16:17:48 -0500942
Jon Hall6aec96b2015-01-19 14:49:31 -0800943 main.step( "Get the flows from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800944 global flowState
945 flowState = []
Jon Hall8f89dda2015-01-22 16:03:33 -0800946 flowCheck = main.FALSE
Jon Hall58c76b72015-02-23 11:09:24 -0800947 try:
948 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
949 ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
950 ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
951 ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
952 ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
953 ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
954 ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
955 assert ONOS1Flows, "ONOS1 Flows should not be empty"
956 assert ONOS2Flows, "ONOS2 Flows should not be empty"
957 assert ONOS3Flows, "ONOS3 Flows should not be empty"
958 assert ONOS4Flows, "ONOS4 Flows should not be empty"
959 assert ONOS5Flows, "ONOS5 Flows should not be empty"
960 assert ONOS6Flows, "ONOS6 Flows should not be empty"
961 assert ONOS7Flows, "ONOS7 Flows should not be empty"
962 assert "Error" not in ONOS1Flows, "ONOS1 Flows contains 'Error'"
963 assert "Error" not in ONOS2Flows, "ONOS2 Flows contains 'Error'"
964 assert "Error" not in ONOS3Flows, "ONOS3 Flows contains 'Error'"
965 assert "Error" not in ONOS4Flows, "ONOS4 Flows contains 'Error'"
966 assert "Error" not in ONOS5Flows, "ONOS5 Flows contains 'Error'"
967 assert "Error" not in ONOS6Flows, "ONOS6 Flows contains 'Error'"
968 assert "Error" not in ONOS7Flows, "ONOS7 Flows contains 'Error'"
969 ONOS1FlowsJson = json.loads( ONOS1Flows )
970 ONOS2FlowsJson = json.loads( ONOS2Flows )
971 ONOS3FlowsJson = json.loads( ONOS3Flows )
972 ONOS4FlowsJson = json.loads( ONOS4Flows )
973 ONOS5FlowsJson = json.loads( ONOS5Flows )
974 ONOS6FlowsJson = json.loads( ONOS6Flows )
975 ONOS7FlowsJson = json.loads( ONOS7Flows )
976 except ( ValueError, AssertionError ): # From json.loads, or asserts
977 main.log.exception( "One or more 'flows' responses from " +
978 "ONOS couldn't be decoded." )
Jon Hall8f89dda2015-01-22 16:03:33 -0800979 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
980 main.log.warn( "ONOS2 flows repsponse: " + ONOS2Flows )
981 main.log.warn( "ONOS3 flows repsponse: " + ONOS3Flows )
982 main.log.warn( "ONOS4 flows repsponse: " + ONOS4Flows )
983 main.log.warn( "ONOS5 flows repsponse: " + ONOS5Flows )
984 main.log.warn( "ONOS6 flows repsponse: " + ONOS6Flows )
985 main.log.warn( "ONOS7 flows repsponse: " + ONOS7Flows )
Jon Hall58c76b72015-02-23 11:09:24 -0800986 else: # No exceptions
987 if len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
988 and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
989 and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
990 and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
991 and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
992 and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
Jon Hall6aec96b2015-01-19 14:49:31 -0800993 # TODO: Do a better check, maybe compare flows on switches?
Jon Hall58c76b72015-02-23 11:09:24 -0800994 # NOTE Possible issue with this not always being set?
995 flowState = ONOS1Flows
996 flowCheck = main.TRUE
997 main.log.report( "Flow count is consistent across all" +
998 " ONOS nodes" )
999 else:
1000 main.log.warn( "ONOS1 flows: " +
1001 json.dumps( ONOS1FlowsJson, sort_keys=True,
1002 indent=4,
1003 separators=( ',', ': ' ) ) )
1004 main.log.warn( "ONOS2 flows: " +
1005 json.dumps( ONOS2FlowsJson, sort_keys=True,
1006 indent=4,
1007 separators=( ',', ': ' ) ) )
1008 main.log.warn( "ONOS3 flows: " +
1009 json.dumps( ONOS3FlowsJson, sort_keys=True,
1010 indent=4,
1011 separators=( ',', ': ' ) ) )
1012 main.log.warn( "ONOS4 flows: " +
1013 json.dumps( ONOS4FlowsJson, sort_keys=True,
1014 indent=4,
1015 separators=( ',', ': ' ) ) )
1016 main.log.warn( "ONOS5 flows: " +
1017 json.dumps( ONOS5FlowsJson, sort_keys=True,
1018 indent=4,
1019 separators=( ',', ': ' ) ) )
1020 main.log.warn( "ONOS6 flows: " +
1021 json.dumps( ONOS6FlowsJson, sort_keys=True,
1022 indent=4,
1023 separators=( ',', ': ' ) ) )
1024 main.log.warn( "ONOS7 flows: " +
1025 json.dumps( ONOS7FlowsJson, sort_keys=True,
1026 indent=4,
1027 separators=( ',', ': ' ) ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001028 utilities.assert_equals(
1029 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001030 actual=flowCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001031 onpass="The flow count is consistent across all ONOS nodes",
1032 onfail="ONOS nodes have different flow counts" )
Jon Hallb1290e82014-11-18 16:17:48 -05001033
Jon Hall6aec96b2015-01-19 14:49:31 -08001034 main.step( "Get the OF Table entries" )
Jon Hallb1290e82014-11-18 16:17:48 -05001035 global flows
Jon Hall6aec96b2015-01-19 14:49:31 -08001036 flows = []
1037 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001038 flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001039 if flowCheck == main.FALSE:
1040 for table in flows:
1041 main.log.warn( table )
Jon Hall6aec96b2015-01-19 14:49:31 -08001042 # TODO: Compare switch flow tables with ONOS flow tables
Jon Hallb1290e82014-11-18 16:17:48 -05001043
Jon Hall6aec96b2015-01-19 14:49:31 -08001044 main.step( "Start continuous pings" )
1045 main.Mininet2.pingLong(
1046 src=main.params[ 'PING' ][ 'source1' ],
1047 target=main.params[ 'PING' ][ 'target1' ],
1048 pingTime=500 )
1049 main.Mininet2.pingLong(
1050 src=main.params[ 'PING' ][ 'source2' ],
1051 target=main.params[ 'PING' ][ 'target2' ],
1052 pingTime=500 )
1053 main.Mininet2.pingLong(
1054 src=main.params[ 'PING' ][ 'source3' ],
1055 target=main.params[ 'PING' ][ 'target3' ],
1056 pingTime=500 )
1057 main.Mininet2.pingLong(
1058 src=main.params[ 'PING' ][ 'source4' ],
1059 target=main.params[ 'PING' ][ 'target4' ],
1060 pingTime=500 )
1061 main.Mininet2.pingLong(
1062 src=main.params[ 'PING' ][ 'source5' ],
1063 target=main.params[ 'PING' ][ 'target5' ],
1064 pingTime=500 )
1065 main.Mininet2.pingLong(
1066 src=main.params[ 'PING' ][ 'source6' ],
1067 target=main.params[ 'PING' ][ 'target6' ],
1068 pingTime=500 )
1069 main.Mininet2.pingLong(
1070 src=main.params[ 'PING' ][ 'source7' ],
1071 target=main.params[ 'PING' ][ 'target7' ],
1072 pingTime=500 )
1073 main.Mininet2.pingLong(
1074 src=main.params[ 'PING' ][ 'source8' ],
1075 target=main.params[ 'PING' ][ 'target8' ],
1076 pingTime=500 )
1077 main.Mininet2.pingLong(
1078 src=main.params[ 'PING' ][ 'source9' ],
1079 target=main.params[ 'PING' ][ 'target9' ],
1080 pingTime=500 )
1081 main.Mininet2.pingLong(
1082 src=main.params[ 'PING' ][ 'source10' ],
1083 target=main.params[ 'PING' ][ 'target10' ],
1084 pingTime=500 )
Jon Hallb1290e82014-11-18 16:17:48 -05001085
Jon Hall6aec96b2015-01-19 14:49:31 -08001086 main.step( "Create TestONTopology object" )
Jon Hallb1290e82014-11-18 16:17:48 -05001087 ctrls = []
1088 count = 1
1089 while True:
1090 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -08001091 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
1092 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
1093 temp = temp + ( "ONOS" + str( count ), )
1094 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
1095 temp = temp + \
1096 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
1097 ctrls.append( temp )
Jon Hallb1290e82014-11-18 16:17:48 -05001098 count = count + 1
1099 else:
1100 break
Jon Hall6aec96b2015-01-19 14:49:31 -08001101 MNTopo = TestONTopology(
1102 main.Mininet1,
1103 ctrls ) # can also add Intent API info for intent operations
Jon Hallb1290e82014-11-18 16:17:48 -05001104
Jon Hall6aec96b2015-01-19 14:49:31 -08001105 main.step( "Collecting topology information from ONOS" )
Jon Hallb1290e82014-11-18 16:17:48 -05001106 devices = []
1107 devices.append( main.ONOScli1.devices() )
1108 devices.append( main.ONOScli2.devices() )
1109 devices.append( main.ONOScli3.devices() )
1110 devices.append( main.ONOScli4.devices() )
1111 devices.append( main.ONOScli5.devices() )
1112 devices.append( main.ONOScli6.devices() )
1113 devices.append( main.ONOScli7.devices() )
Jon Hallb1290e82014-11-18 16:17:48 -05001114 hosts = []
Jon Hall58c76b72015-02-23 11:09:24 -08001115 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1116 hosts.append( json.loads( main.ONOScli2.hosts() ) )
1117 hosts.append( json.loads( main.ONOScli3.hosts() ) )
1118 hosts.append( json.loads( main.ONOScli4.hosts() ) )
1119 hosts.append( json.loads( main.ONOScli5.hosts() ) )
1120 hosts.append( json.loads( main.ONOScli6.hosts() ) )
1121 hosts.append( json.loads( main.ONOScli7.hosts() ) )
Jon Hallb1290e82014-11-18 16:17:48 -05001122 ports = []
1123 ports.append( main.ONOScli1.ports() )
1124 ports.append( main.ONOScli2.ports() )
1125 ports.append( main.ONOScli3.ports() )
1126 ports.append( main.ONOScli4.ports() )
1127 ports.append( main.ONOScli5.ports() )
1128 ports.append( main.ONOScli6.ports() )
1129 ports.append( main.ONOScli7.ports() )
1130 links = []
1131 links.append( main.ONOScli1.links() )
1132 links.append( main.ONOScli2.links() )
1133 links.append( main.ONOScli3.links() )
1134 links.append( main.ONOScli4.links() )
1135 links.append( main.ONOScli5.links() )
1136 links.append( main.ONOScli6.links() )
1137 links.append( main.ONOScli7.links() )
Jon Hall94fd0472014-12-08 11:52:42 -08001138 clusters = []
1139 clusters.append( main.ONOScli1.clusters() )
1140 clusters.append( main.ONOScli2.clusters() )
1141 clusters.append( main.ONOScli3.clusters() )
1142 clusters.append( main.ONOScli4.clusters() )
1143 clusters.append( main.ONOScli5.clusters() )
1144 clusters.append( main.ONOScli6.clusters() )
1145 clusters.append( main.ONOScli7.clusters() )
Jon Hall529a37f2015-01-28 10:02:00 -08001146 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001147
Jon Hall6aec96b2015-01-19 14:49:31 -08001148 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -08001149 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001150 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001151 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001152 if "Error" not in hosts[ controller ]:
1153 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001154 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001155 else: # hosts not consistent
1156 main.log.report( "hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001157 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001158 " is inconsistent with ONOS1" )
1159 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001160 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001161
1162 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001163 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001164 controllerStr )
1165 consistentHostsResult = main.FALSE
1166 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001167 " hosts response: " +
1168 repr( hosts[ controller ] ) )
1169 utilities.assert_equals(
1170 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001171 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001172 onpass="Hosts view is consistent across all ONOS nodes",
1173 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001174
Jon Hall58c76b72015-02-23 11:09:24 -08001175 ipResult = main.TRUE
1176 for controller in range( 0, len( hosts ) ):
1177 controllerStr = str( controller + 1 )
1178 for host in hosts[ controller ]:
1179 if host.get( 'ips', [] ) == []:
1180 main.log.error(
1181 "DEBUG:Error with host ips on controller" +
1182 controllerStr + ": " + str( host ) )
1183 ipResult = main.FALSE
1184 utilities.assert_equals(
1185 expect=main.TRUE,
1186 actual=ipResult,
1187 onpass="The ips of the hosts aren't empty",
1188 onfail="The ip of at least one host is missing" )
1189
Jon Hall6aec96b2015-01-19 14:49:31 -08001190 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001191 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001192 for controller in range( len( clusters ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001193 if "Error" not in clusters[ controller ]:
1194 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001195 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001196 else: # clusters not consistent
1197 main.log.report( "clusters from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001198 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001199 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001200 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001201
1202 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001203 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001204 "from ONOS" + controllerStr )
1205 consistentClustersResult = main.FALSE
1206 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001207 " clusters response: " +
1208 repr( clusters[ controller ] ) )
1209 utilities.assert_equals(
1210 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001211 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001212 onpass="Clusters view is consistent across all ONOS nodes",
1213 onfail="ONOS nodes have different views of clusters" )
1214 # there should always only be one cluster
Jon Hall8f89dda2015-01-22 16:03:33 -08001215 numClusters = len( json.loads( clusters[ 0 ] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001216 clusterResults = main.FALSE
1217 if numClusters == 1:
1218 clusterResults = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001219 utilities.assert_equals(
1220 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001221 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001222 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08001223 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001224
Jon Hall6aec96b2015-01-19 14:49:31 -08001225 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001226 devicesResults = main.TRUE
1227 portsResults = main.TRUE
1228 linksResults = main.TRUE
1229 for controller in range( numControllers ):
1230 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001231 if devices[ controller ] or "Error" not in devices[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001232 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001233 MNTopo,
1234 json.loads(
1235 devices[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001236 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001237 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001238 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001239 actual=currentDevicesResult,
1240 onpass="ONOS" + controllerStr +
1241 " Switches view is correct",
1242 onfail="ONOS" + controllerStr +
1243 " Switches view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001244
Jon Hall6aec96b2015-01-19 14:49:31 -08001245 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001246 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001247 MNTopo,
1248 json.loads(
1249 ports[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001250 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001251 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001252 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001253 actual=currentPortsResult,
1254 onpass="ONOS" + controllerStr +
1255 " ports view is correct",
1256 onfail="ONOS" + controllerStr +
1257 " ports view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001258
Jon Hall6aec96b2015-01-19 14:49:31 -08001259 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001260 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001261 MNTopo,
1262 json.loads(
1263 links[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001264 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001265 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001266 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001267 actual=currentLinksResult,
1268 onpass="ONOS" + controllerStr +
1269 " links view is correct",
1270 onfail="ONOS" + controllerStr +
1271 " links view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001272
Jon Hall8f89dda2015-01-22 16:03:33 -08001273 devicesResults = devicesResults and currentDevicesResult
1274 portsResults = portsResults and currentPortsResult
1275 linksResults = linksResults and currentLinksResult
Jon Hallb1290e82014-11-18 16:17:48 -05001276
Jon Hall8f89dda2015-01-22 16:03:33 -08001277 topoResult = devicesResults and portsResults and linksResults\
Jon Hall58c76b72015-02-23 11:09:24 -08001278 and consistentHostsResult and consistentClustersResult\
1279 and clusterResults and ipResult
Jon Hall8f89dda2015-01-22 16:03:33 -08001280 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08001281 onpass="Topology Check Test successful",
1282 onfail="Topology Check Test NOT successful" )
Jon Hallb1290e82014-11-18 16:17:48 -05001283
Jon Hall8f89dda2015-01-22 16:03:33 -08001284 finalAssert = main.TRUE
1285 finalAssert = finalAssert and topoResult and flowCheck \
Jon Hall58c76b72015-02-23 11:09:24 -08001286 and intentCheck and consistentMastership and rolesNotNull
Jon Hall8f89dda2015-01-22 16:03:33 -08001287 utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
Jon Hall58c76b72015-02-23 11:09:24 -08001288 onpass="State check successful",
1289 onfail="State check NOT successful" )
Jon Hallb1290e82014-11-18 16:17:48 -05001290
Jon Hall6aec96b2015-01-19 14:49:31 -08001291 def CASE6( self, main ):
1292 """
Jon Hallb1290e82014-11-18 16:17:48 -05001293 The Failure case. Since this is the Sanity test, we do nothing.
Jon Hall6aec96b2015-01-19 14:49:31 -08001294 """
Jon Hall368769f2014-11-19 15:43:35 -08001295 import time
Jon Hall6aec96b2015-01-19 14:49:31 -08001296 main.log.report( "Wait 60 seconds instead of inducing a failure" )
1297 time.sleep( 60 )
1298 utilities.assert_equals(
1299 expect=main.TRUE,
1300 actual=main.TRUE,
1301 onpass="Sleeping 60 seconds",
1302 onfail="Something is terribly wrong with my math" )
Jon Hallb1290e82014-11-18 16:17:48 -05001303
Jon Hall6aec96b2015-01-19 14:49:31 -08001304 def CASE7( self, main ):
1305 """
Jon Hall368769f2014-11-19 15:43:35 -08001306 Check state after ONOS failure
Jon Hall6aec96b2015-01-19 14:49:31 -08001307 """
Jon Hallb1290e82014-11-18 16:17:48 -05001308 import json
Jon Hall6aec96b2015-01-19 14:49:31 -08001309 main.case( "Running ONOS Constant State Tests" )
Jon Hallb1290e82014-11-18 16:17:48 -05001310
Jon Hall6aec96b2015-01-19 14:49:31 -08001311 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -08001312 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
1313 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
1314 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
1315 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
1316 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
1317 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
1318 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
1319 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
1320 ONOS3MasterNotNull and ONOS4MasterNotNull and\
1321 ONOS5MasterNotNull and ONOS6MasterNotNull and\
1322 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -08001323 utilities.assert_equals(
1324 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001325 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -08001326 onpass="Each device has a master",
1327 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -08001328
Jon Hall6aec96b2015-01-19 14:49:31 -08001329 main.step( "Check if switch roles are consistent across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001330 ONOS1Mastership = main.ONOScli1.roles()
1331 ONOS2Mastership = main.ONOScli2.roles()
1332 ONOS3Mastership = main.ONOScli3.roles()
1333 ONOS4Mastership = main.ONOScli4.roles()
1334 ONOS5Mastership = main.ONOScli5.roles()
1335 ONOS6Mastership = main.ONOScli6.roles()
1336 ONOS7Mastership = main.ONOScli7.roles()
Jon Hall8f89dda2015-01-22 16:03:33 -08001337 if "Error" in ONOS1Mastership or not ONOS1Mastership\
1338 or "Error" in ONOS2Mastership or not ONOS2Mastership\
1339 or "Error" in ONOS3Mastership or not ONOS3Mastership\
1340 or "Error" in ONOS4Mastership or not ONOS4Mastership\
1341 or "Error" in ONOS5Mastership or not ONOS5Mastership\
1342 or "Error" in ONOS6Mastership or not ONOS6Mastership\
1343 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -08001344 main.log.error( "Error in getting ONOS mastership" )
1345 main.log.warn( "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001346 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001347 main.log.warn( "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001348 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001349 main.log.warn( "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001350 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001351 main.log.warn( "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001352 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001353 main.log.warn( "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001354 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001355 main.log.warn( "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001356 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001357 main.log.warn( "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001358 repr( ONOS7Mastership ) )
1359 consistentMastership = main.FALSE
1360 elif ONOS1Mastership == ONOS2Mastership\
1361 and ONOS1Mastership == ONOS3Mastership\
1362 and ONOS1Mastership == ONOS4Mastership\
1363 and ONOS1Mastership == ONOS5Mastership\
1364 and ONOS1Mastership == ONOS6Mastership\
1365 and ONOS1Mastership == ONOS7Mastership:
1366 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001367 main.log.report(
1368 "Switch roles are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -05001369 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001370 main.log.warn( "ONOS1 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001371 json.loads( ONOS1Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001372 separators=( ',', ': ' ) ) )
1373 main.log.warn( "ONOS2 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001374 json.loads( ONOS2Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001375 separators=( ',', ': ' ) ) )
1376 main.log.warn( "ONOS3 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001377 json.loads( ONOS3Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001378 separators=( ',', ': ' ) ) )
1379 main.log.warn( "ONOS4 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001380 json.loads( ONOS4Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001381 separators=( ',', ': ' ) ) )
1382 main.log.warn( "ONOS5 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001383 json.loads( ONOS5Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001384 separators=( ',', ': ' ) ) )
1385 main.log.warn( "ONOS6 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001386 json.loads( ONOS6Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001387 separators=( ',', ': ' ) ) )
1388 main.log.warn( "ONOS7 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001389 json.loads( ONOS7Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001390 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001391 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001392 utilities.assert_equals(
1393 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001394 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -08001395 onpass="Switch roles are consistent across all ONOS nodes",
1396 onfail="ONOS nodes have different views of switch roles" )
Jon Hallb1290e82014-11-18 16:17:48 -05001397
1398 description2 = "Compare switch roles from before failure"
Jon Hall6aec96b2015-01-19 14:49:31 -08001399 main.step( description2 )
Jon Hallb1290e82014-11-18 16:17:48 -05001400
Jon Hall8f89dda2015-01-22 16:03:33 -08001401 currentJson = json.loads( ONOS1Mastership )
1402 oldJson = json.loads( mastershipState )
1403 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001404 for i in range( 1, 29 ):
1405 switchDPID = str(
1406 main.Mininet1.getSwitchDPID(
1407 switch="s" +
1408 str( i ) ) )
Jon Hallb1290e82014-11-18 16:17:48 -05001409
Jon Hall8f89dda2015-01-22 16:03:33 -08001410 current = [ switch[ 'master' ] for switch in currentJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001411 if switchDPID in switch[ 'id' ] ]
Jon Hall8f89dda2015-01-22 16:03:33 -08001412 old = [ switch[ 'master' ] for switch in oldJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001413 if switchDPID in switch[ 'id' ] ]
Jon Hallb1290e82014-11-18 16:17:48 -05001414 if current == old:
Jon Hall8f89dda2015-01-22 16:03:33 -08001415 mastershipCheck = mastershipCheck and main.TRUE
Jon Hallb1290e82014-11-18 16:17:48 -05001416 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001417 main.log.warn( "Mastership of switch %s changed" % switchDPID )
Jon Hall8f89dda2015-01-22 16:03:33 -08001418 mastershipCheck = main.FALSE
1419 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001420 main.log.report( "Mastership of Switches was not changed" )
1421 utilities.assert_equals(
1422 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001423 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001424 onpass="Mastership of Switches was not changed",
1425 onfail="Mastership of some switches changed" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001426 mastershipCheck = mastershipCheck and consistentMastership
Jon Hallb1290e82014-11-18 16:17:48 -05001427
Jon Hall6aec96b2015-01-19 14:49:31 -08001428 main.step( "Get the intents and compare across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001429 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1430 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
1431 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
1432 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
1433 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
1434 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
1435 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
1436 intentCheck = main.FALSE
1437 if "Error" in ONOS1Intents or not ONOS1Intents\
1438 or "Error" in ONOS2Intents or not ONOS2Intents\
1439 or "Error" in ONOS3Intents or not ONOS3Intents\
1440 or "Error" in ONOS4Intents or not ONOS4Intents\
1441 or "Error" in ONOS5Intents or not ONOS5Intents\
1442 or "Error" in ONOS6Intents or not ONOS6Intents\
1443 or "Error" in ONOS7Intents or not ONOS7Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -08001444 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001445 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1446 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
1447 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
1448 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
1449 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
1450 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
1451 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
1452 elif ONOS1Intents == ONOS2Intents\
1453 and ONOS1Intents == ONOS3Intents\
1454 and ONOS1Intents == ONOS4Intents\
1455 and ONOS1Intents == ONOS5Intents\
1456 and ONOS1Intents == ONOS6Intents\
1457 and ONOS1Intents == ONOS7Intents:
1458 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001459 main.log.report( "Intents are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -05001460 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001461 main.log.warn( "ONOS1 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001462 print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001463 indent=4, separators=( ',', ': ' ) )
1464 main.log.warn( "ONOS2 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001465 print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001466 indent=4, separators=( ',', ': ' ) )
1467 main.log.warn( "ONOS3 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001468 print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001469 indent=4, separators=( ',', ': ' ) )
1470 main.log.warn( "ONOS4 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001471 print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001472 indent=4, separators=( ',', ': ' ) )
1473 main.log.warn( "ONOS5 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001474 print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001475 indent=4, separators=( ',', ': ' ) )
1476 main.log.warn( "ONOS6 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001477 print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001478 indent=4, separators=( ',', ': ' ) )
1479 main.log.warn( "ONOS7 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001480 print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001481 indent=4, separators=( ',', ': ' ) )
1482 utilities.assert_equals(
1483 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001484 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001485 onpass="Intents are consistent across all ONOS nodes",
1486 onfail="ONOS nodes have different views of intents" )
Jon Hall1b8f54a2015-02-04 13:24:20 -08001487 # Print the intent states
1488 intents = []
1489 intents.append( ONOS1Intents )
1490 intents.append( ONOS2Intents )
1491 intents.append( ONOS3Intents )
1492 intents.append( ONOS4Intents )
1493 intents.append( ONOS5Intents )
1494 intents.append( ONOS6Intents )
1495 intents.append( ONOS7Intents )
1496 intentStates = []
1497 for node in intents: # Iter through ONOS nodes
1498 nodeStates = []
Jon Hall58c76b72015-02-23 11:09:24 -08001499 # Iter through intents of a node
1500 for intent in json.loads( node ):
Jon Hall1b8f54a2015-02-04 13:24:20 -08001501 nodeStates.append( intent[ 'state' ] )
1502 intentStates.append( nodeStates )
1503 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1504 main.log.info( dict( out ) )
1505
Jon Hall58c76b72015-02-23 11:09:24 -08001506 # NOTE: Store has no durability, so intents are lost across system
1507 # restarts
Jon Hall6aec96b2015-01-19 14:49:31 -08001508 main.step( "Compare current intents with intents before the failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001509 # NOTE: this requires case 5 to pass for intentState to be set.
Jon Hall94fd0472014-12-08 11:52:42 -08001510 # maybe we should stop the test if that fails?
Jon Hall1b8f54a2015-02-04 13:24:20 -08001511 sameIntents = main.TRUE
1512 if intentState and intentState == ONOS1Intents:
Jon Hall8f89dda2015-01-22 16:03:33 -08001513 sameIntents = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001514 main.log.report( "Intents are consistent with before failure" )
1515 # TODO: possibly the states have changed? we may need to figure out
1516 # what the aceptable states are
Jon Hallb1290e82014-11-18 16:17:48 -05001517 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001518 try:
1519 main.log.warn( "ONOS1 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001520 print json.dumps( json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -08001521 sort_keys=True, indent=4,
1522 separators=( ',', ': ' ) )
Jon Hallfebb1c72015-03-05 13:30:09 -08001523 except Exception:
Jon Hall6aec96b2015-01-19 14:49:31 -08001524 pass
Jon Hall8f89dda2015-01-22 16:03:33 -08001525 sameIntents = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001526 utilities.assert_equals(
1527 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001528 actual=sameIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001529 onpass="Intents are consistent with before failure",
1530 onfail="The Intents changed during failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001531 intentCheck = intentCheck and sameIntents
Jon Hallb1290e82014-11-18 16:17:48 -05001532
Jon Hall6aec96b2015-01-19 14:49:31 -08001533 main.step( "Get the OF Table entries and compare to before " +
1534 "component failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001535 FlowTables = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001536 flows2 = []
1537 for i in range( 28 ):
1538 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001539 tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
1540 flows2.append( tmpFlows )
1541 tempResult = main.Mininet2.flowComp(
Jon Hall6aec96b2015-01-19 14:49:31 -08001542 flow1=flows[ i ],
Jon Hall8f89dda2015-01-22 16:03:33 -08001543 flow2=tmpFlows )
1544 FlowTables = FlowTables and tempResult
1545 if FlowTables == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001546 main.log.info( "Differences in flow table for switch: s" +
1547 str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001548 if FlowTables == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001549 main.log.report( "No changes were found in the flow tables" )
1550 utilities.assert_equals(
1551 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001552 actual=FlowTables,
Jon Hall6aec96b2015-01-19 14:49:31 -08001553 onpass="No changes were found in the flow tables",
1554 onfail="Changes were found in the flow tables" )
Jon Hallb1290e82014-11-18 16:17:48 -05001555
Jon Hall6aec96b2015-01-19 14:49:31 -08001556 main.step( "Check the continuous pings to ensure that no packets " +
1557 "were dropped during component failure" )
1558 # FIXME: This check is always failing. Investigate cause
1559 # NOTE: this may be something to do with file permsissions
Jon Hallb1290e82014-11-18 16:17:48 -05001560 # or slight change in format
Jon Hall6aec96b2015-01-19 14:49:31 -08001561 main.Mininet2.pingKill(
1562 main.params[ 'TESTONUSER' ],
1563 main.params[ 'TESTONIP' ] )
Jon Hall8f89dda2015-01-22 16:03:33 -08001564 LossInPings = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001565 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1566 for i in range( 8, 18 ):
1567 main.log.info(
1568 "Checking for a loss in pings along flow from s" +
1569 str( i ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001570 LossInPings = main.Mininet2.checkForLoss(
Jon Hall6aec96b2015-01-19 14:49:31 -08001571 "/tmp/ping.h" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001572 str( i ) ) or LossInPings
1573 if LossInPings == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001574 main.log.info( "Loss in ping detected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001575 elif LossInPings == main.ERROR:
Jon Hall6aec96b2015-01-19 14:49:31 -08001576 main.log.info( "There are multiple mininet process running" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001577 elif LossInPings == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001578 main.log.info( "No Loss in the pings" )
1579 main.log.report( "No loss of dataplane connectivity" )
1580 utilities.assert_equals(
1581 expect=main.FALSE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001582 actual=LossInPings,
Jon Hall6aec96b2015-01-19 14:49:31 -08001583 onpass="No Loss of connectivity",
1584 onfail="Loss of dataplane connectivity detected" )
Jon Hallb1290e82014-11-18 16:17:48 -05001585
Jon Hall6aec96b2015-01-19 14:49:31 -08001586 # Test of LeadershipElection
1587 # NOTE: this only works for the sanity test. In case of failures,
Jon Hall58c76b72015-02-23 11:09:24 -08001588 # leader will likely change
Jon Hall8f89dda2015-01-22 16:03:33 -08001589 leader = ONOS1Ip
1590 leaderResult = main.TRUE
1591 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001592 # loop through ONOScli handlers
1593 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001594 leaderN = node.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001595 # verify leader is ONOS1
Jon Hall669173b2014-12-17 11:36:30 -08001596 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08001597 # all is well
1598 # NOTE: In failure scenario, this could be a new node, maybe
1599 # check != ONOS1
Jon Hall669173b2014-12-17 11:36:30 -08001600 pass
1601 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001602 # error in response
1603 main.log.report( "Something is wrong with " +
Jon Hall58c76b72015-02-23 11:09:24 -08001604 "electionTestLeader function, check the" +
1605 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001606 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001607 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08001608 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001609 main.log.report( "ONOS" + str( controller ) + " sees " +
1610 str( leaderN ) +
1611 " as the leader of the election app. " +
1612 "Leader should be " + str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001613 if leaderResult:
1614 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001615 "view of leader across listeners and a new " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001616 "leader was re-elected if applicable )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001617 utilities.assert_equals(
1618 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001619 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001620 onpass="Leadership election passed",
1621 onfail="Something went wrong with Leadership election" )
Jon Hallb1290e82014-11-18 16:17:48 -05001622
Jon Hall58c76b72015-02-23 11:09:24 -08001623 result = ( mastershipCheck and intentCheck and FlowTables and
1624 ( not LossInPings ) and rolesNotNull and leaderResult )
Jon Hall6aec96b2015-01-19 14:49:31 -08001625 result = int( result )
Jon Hallb1290e82014-11-18 16:17:48 -05001626 if result == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001627 main.log.report( "Constant State Tests Passed" )
1628 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08001629 onpass="Constant State Tests Passed",
1630 onfail="Constant state tests failed" )
Jon Hallb1290e82014-11-18 16:17:48 -05001631
Jon Hall6aec96b2015-01-19 14:49:31 -08001632 def CASE8( self, main ):
1633 """
Jon Hallb1290e82014-11-18 16:17:48 -05001634 Compare topo
Jon Hall6aec96b2015-01-19 14:49:31 -08001635 """
Jon Hallb1290e82014-11-18 16:17:48 -05001636 import sys
Jon Hall6aec96b2015-01-19 14:49:31 -08001637 # FIXME add this path to params
1638 sys.path.append( "/home/admin/sts" )
1639 # assumes that sts is already in you PYTHONPATH
1640 from sts.topology.teston_topology import TestONTopology
Jon Hallb1290e82014-11-18 16:17:48 -05001641 import json
Jon Hall73cf9cc2014-11-20 22:28:38 -08001642 import time
Jon Hallb1290e82014-11-18 16:17:48 -05001643
Jon Hall6aec96b2015-01-19 14:49:31 -08001644 description = "Compare ONOS Topology view to Mininet topology"
1645 main.case( description )
1646 main.log.report( description )
1647 main.step( "Create TestONTopology object" )
Jon Hallb1290e82014-11-18 16:17:48 -05001648 ctrls = []
1649 count = 1
1650 while True:
1651 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -08001652 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
1653 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
1654 temp = temp + ( "ONOS" + str( count ), )
1655 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
1656 temp = temp + \
1657 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
1658 ctrls.append( temp )
Jon Hallb1290e82014-11-18 16:17:48 -05001659 count = count + 1
1660 else:
1661 break
Jon Hall6aec96b2015-01-19 14:49:31 -08001662 MNTopo = TestONTopology(
1663 main.Mininet1,
1664 ctrls ) # can also add Intent API info for intent operations
Jon Hallb1290e82014-11-18 16:17:48 -05001665
Jon Hall6aec96b2015-01-19 14:49:31 -08001666 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001667 devicesResults = main.TRUE
1668 portsResults = main.TRUE
1669 linksResults = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -08001670 hostsResults = main.TRUE
Jon Hall8f89dda2015-01-22 16:03:33 -08001671 topoResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001672 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -08001673 count = 0
Jon Hall6aec96b2015-01-19 14:49:31 -08001674 main.step( "Collecting topology information from ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001675 startTime = time.time()
Jon Hall21270ac2015-02-16 17:59:55 -08001676 # Give time for Gossip to work
Jon Hall8f89dda2015-01-22 16:03:33 -08001677 while topoResult == main.FALSE and elapsed < 60:
Jon Hallffb386d2014-11-21 13:43:38 -08001678 count = count + 1
Jon Hall94fd0472014-12-08 11:52:42 -08001679 if count > 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08001680 # TODO: Depricate STS usage
Jon Hall58c76b72015-02-23 11:09:24 -08001681 MNTopo = TestONTopology( main.Mininet1, ctrls )
Jon Hall8f89dda2015-01-22 16:03:33 -08001682 cliStart = time.time()
Jon Hall94fd0472014-12-08 11:52:42 -08001683 devices = []
1684 devices.append( main.ONOScli1.devices() )
1685 devices.append( main.ONOScli2.devices() )
1686 devices.append( main.ONOScli3.devices() )
1687 devices.append( main.ONOScli4.devices() )
1688 devices.append( main.ONOScli5.devices() )
1689 devices.append( main.ONOScli6.devices() )
1690 devices.append( main.ONOScli7.devices() )
1691 hosts = []
Jon Hall529a37f2015-01-28 10:02:00 -08001692 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1693 hosts.append( json.loads( main.ONOScli2.hosts() ) )
1694 hosts.append( json.loads( main.ONOScli3.hosts() ) )
1695 hosts.append( json.loads( main.ONOScli4.hosts() ) )
1696 hosts.append( json.loads( main.ONOScli5.hosts() ) )
1697 hosts.append( json.loads( main.ONOScli6.hosts() ) )
1698 hosts.append( json.loads( main.ONOScli7.hosts() ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001699 ipResult = main.TRUE
Jon Hall529a37f2015-01-28 10:02:00 -08001700 for controller in range( 0, len( hosts ) ):
1701 controllerStr = str( controller + 1 )
1702 for host in hosts[ controller ]:
Jon Hall58c76b72015-02-23 11:09:24 -08001703 if host is None or host.get( 'ips', [] ) == []:
Jon Hall529a37f2015-01-28 10:02:00 -08001704 main.log.error(
1705 "DEBUG:Error with host ips on controller" +
1706 controllerStr + ": " + str( host ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001707 ipResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001708 ports = []
1709 ports.append( main.ONOScli1.ports() )
1710 ports.append( main.ONOScli2.ports() )
1711 ports.append( main.ONOScli3.ports() )
1712 ports.append( main.ONOScli4.ports() )
1713 ports.append( main.ONOScli5.ports() )
1714 ports.append( main.ONOScli6.ports() )
1715 ports.append( main.ONOScli7.ports() )
1716 links = []
1717 links.append( main.ONOScli1.links() )
1718 links.append( main.ONOScli2.links() )
1719 links.append( main.ONOScli3.links() )
1720 links.append( main.ONOScli4.links() )
1721 links.append( main.ONOScli5.links() )
1722 links.append( main.ONOScli6.links() )
1723 links.append( main.ONOScli7.links() )
1724 clusters = []
1725 clusters.append( main.ONOScli1.clusters() )
1726 clusters.append( main.ONOScli2.clusters() )
1727 clusters.append( main.ONOScli3.clusters() )
1728 clusters.append( main.ONOScli4.clusters() )
1729 clusters.append( main.ONOScli5.clusters() )
1730 clusters.append( main.ONOScli6.clusters() )
1731 clusters.append( main.ONOScli7.clusters() )
Jon Hallb1290e82014-11-18 16:17:48 -05001732
Jon Hall8f89dda2015-01-22 16:03:33 -08001733 elapsed = time.time() - startTime
1734 cliTime = time.time() - cliStart
1735 print "CLI time: " + str( cliTime )
Jon Hallb1290e82014-11-18 16:17:48 -05001736
Jon Hall21270ac2015-02-16 17:59:55 -08001737 for controller in range( numControllers ):
1738 controllerStr = str( controller + 1 )
1739 if devices[ controller ] or "Error" not in devices[
1740 controller ]:
1741 currentDevicesResult = main.Mininet1.compareSwitches(
1742 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001743 json.loads( devices[ controller ] ) )
Jon Hall21270ac2015-02-16 17:59:55 -08001744 else:
1745 currentDevicesResult = main.FALSE
1746 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001747 actual=currentDevicesResult,
1748 onpass="ONOS" + controllerStr +
1749 " Switches view is correct",
1750 onfail="ONOS" + controllerStr +
1751 " Switches view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001752
Jon Hall21270ac2015-02-16 17:59:55 -08001753 if ports[ controller ] or "Error" not in ports[ controller ]:
1754 currentPortsResult = main.Mininet1.comparePorts(
1755 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001756 json.loads( ports[ controller ] ) )
Jon Hall21270ac2015-02-16 17:59:55 -08001757 else:
1758 currentPortsResult = main.FALSE
1759 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001760 actual=currentPortsResult,
1761 onpass="ONOS" + controllerStr +
1762 " ports view is correct",
1763 onfail="ONOS" + controllerStr +
1764 " ports view is incorrect" )
Jon Hall94fd0472014-12-08 11:52:42 -08001765
Jon Hall21270ac2015-02-16 17:59:55 -08001766 if links[ controller ] or "Error" not in links[ controller ]:
1767 currentLinksResult = main.Mininet1.compareLinks(
1768 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001769 json.loads( links[ controller ] ) )
Jon Hall21270ac2015-02-16 17:59:55 -08001770 else:
1771 currentLinksResult = main.FALSE
1772 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001773 actual=currentLinksResult,
1774 onpass="ONOS" + controllerStr +
1775 " links view is correct",
1776 onfail="ONOS" + controllerStr +
1777 " links view is incorrect" )
1778
1779 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1780 currentHostsResult = main.Mininet1.compareHosts(
1781 MNTopo, hosts[ controller ] )
1782 else:
1783 currentHostsResult = main.FALSE
1784 utilities.assert_equals( expect=main.TRUE,
1785 actual=currentHostsResult,
1786 onpass="ONOS" + controllerStr +
1787 " hosts exist in Mininet",
1788 onfail="ONOS" + controllerStr +
1789 " hosts don't match Mininet" )
1790
1791 devicesResults = devicesResults and currentDevicesResult
1792 portsResults = portsResults and currentPortsResult
1793 linksResults = linksResults and currentLinksResult
1794 hostsResults = hostsResults and currentHostsResult
Jon Hall94fd0472014-12-08 11:52:42 -08001795
Jon Hall21270ac2015-02-16 17:59:55 -08001796 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001797
Jon Hall21270ac2015-02-16 17:59:55 -08001798 # hosts
1799 consistentHostsResult = main.TRUE
1800 for controller in range( len( hosts ) ):
1801 controllerStr = str( controller + 1 )
1802 if "Error" not in hosts[ controller ]:
1803 if hosts[ controller ] == hosts[ 0 ]:
1804 continue
1805 else: # hosts not consistent
1806 main.log.report( "hosts from ONOS" + controllerStr +
1807 " is inconsistent with ONOS1" )
1808 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001809 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001810
Jon Hall21270ac2015-02-16 17:59:55 -08001811 else:
1812 main.log.report( "Error in getting ONOS hosts from ONOS" +
1813 controllerStr )
1814 consistentHostsResult = main.FALSE
1815 main.log.warn( "ONOS" + controllerStr +
1816 " hosts response: " +
1817 repr( hosts[ controller ] ) )
1818 utilities.assert_equals(
1819 expect=main.TRUE,
1820 actual=consistentHostsResult,
1821 onpass="Hosts view is consistent across all ONOS nodes",
1822 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001823
Jon Hall21270ac2015-02-16 17:59:55 -08001824 # Strongly connected clusters of devices
1825 consistentClustersResult = main.TRUE
1826 for controller in range( len( clusters ) ):
1827 controllerStr = str( controller + 1 )
1828 if "Error" not in clusters[ controller ]:
1829 if clusters[ controller ] == clusters[ 0 ]:
1830 continue
1831 else: # clusters not consistent
1832 main.log.report( "clusters from ONOS" +
1833 controllerStr +
1834 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001835 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001836
Jon Hall21270ac2015-02-16 17:59:55 -08001837 else:
1838 main.log.report( "Error in getting dataplane clusters " +
1839 "from ONOS" + controllerStr )
1840 consistentClustersResult = main.FALSE
1841 main.log.warn( "ONOS" + controllerStr +
1842 " clusters response: " +
1843 repr( clusters[ controller ] ) )
1844 utilities.assert_equals(
1845 expect=main.TRUE,
1846 actual=consistentClustersResult,
1847 onpass="Clusters view is consistent across all ONOS nodes",
1848 onfail="ONOS nodes have different views of clusters" )
1849 # there should always only be one cluster
1850 numClusters = len( json.loads( clusters[ 0 ] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001851 clusterResults = main.FALSE
1852 if numClusters == 1:
1853 clusterResults = main.TRUE
Jon Hall21270ac2015-02-16 17:59:55 -08001854 utilities.assert_equals(
1855 expect=1,
1856 actual=numClusters,
1857 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08001858 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001859
Jon Hall21270ac2015-02-16 17:59:55 -08001860 topoResult = ( devicesResults and portsResults and linksResults
Jon Hall58c76b72015-02-23 11:09:24 -08001861 and hostsResults and consistentHostsResult
1862 and consistentClustersResult and clusterResults
1863 and ipResult )
Jon Hall94fd0472014-12-08 11:52:42 -08001864
Jon Hall21270ac2015-02-16 17:59:55 -08001865 topoResult = topoResult and int( count <= 2 )
1866 note = "note it takes about " + str( int( cliTime ) ) + \
1867 " seconds for the test to make all the cli calls to fetch " +\
1868 "the topology from each ONOS instance"
1869 main.log.info(
1870 "Very crass estimate for topology discovery/convergence( " +
1871 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1872 str( count ) + " tries" )
1873 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08001874 onpass="Topology Check Test successful",
1875 onfail="Topology Check Test NOT successful" )
Jon Hall21270ac2015-02-16 17:59:55 -08001876 if topoResult == main.TRUE:
1877 main.log.report( "ONOS topology view matches Mininet topology" )
Jon Hallb1290e82014-11-18 16:17:48 -05001878
Jon Hall6aec96b2015-01-19 14:49:31 -08001879 def CASE9( self, main ):
1880 """
Jon Hallb1290e82014-11-18 16:17:48 -05001881 Link s3-s28 down
Jon Hall6aec96b2015-01-19 14:49:31 -08001882 """
1883 import time
1884 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05001885
Jon Hall8f89dda2015-01-22 16:03:33 -08001886 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hallb1290e82014-11-18 16:17:48 -05001887
Jon Hall6aec96b2015-01-19 14:49:31 -08001888 description = "Turn off a link to ensure that Link Discovery " +\
Jon Hall58c76b72015-02-23 11:09:24 -08001889 "is working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001890 main.log.report( description )
1891 main.case( description )
Jon Hallb1290e82014-11-18 16:17:48 -05001892
Jon Hall6aec96b2015-01-19 14:49:31 -08001893 main.step( "Kill Link between s3 and s28" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001894 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
Jon Hall58c76b72015-02-23 11:09:24 -08001895 main.log.info( "Waiting " + str( linkSleep ) +
1896 " seconds for link down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001897 time.sleep( linkSleep )
1898 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
Jon Hall58c76b72015-02-23 11:09:24 -08001899 onpass="Link down succesful",
1900 onfail="Failed to bring link down" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001901 # TODO do some sort of check here
Jon Hallb1290e82014-11-18 16:17:48 -05001902
Jon Hall6aec96b2015-01-19 14:49:31 -08001903 def CASE10( self, main ):
1904 """
Jon Hallb1290e82014-11-18 16:17:48 -05001905 Link s3-s28 up
Jon Hall6aec96b2015-01-19 14:49:31 -08001906 """
1907 import time
1908 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05001909
Jon Hall8f89dda2015-01-22 16:03:33 -08001910 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hallb1290e82014-11-18 16:17:48 -05001911
Jon Hall6aec96b2015-01-19 14:49:31 -08001912 description = "Restore a link to ensure that Link Discovery is " + \
Jon Hall63604932015-02-26 17:09:50 -08001913 "working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001914 main.log.report( description )
1915 main.case( description )
Jon Hallb1290e82014-11-18 16:17:48 -05001916
Jon Hall6aec96b2015-01-19 14:49:31 -08001917 main.step( "Bring link between s3 and s28 back up" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001918 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
Jon Hall58c76b72015-02-23 11:09:24 -08001919 main.log.info( "Waiting " + str( linkSleep ) +
1920 " seconds for link up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001921 time.sleep( linkSleep )
1922 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
Jon Hall58c76b72015-02-23 11:09:24 -08001923 onpass="Link up succesful",
1924 onfail="Failed to bring link up" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001925 # TODO do some sort of check here
Jon Hallb1290e82014-11-18 16:17:48 -05001926
Jon Hall6aec96b2015-01-19 14:49:31 -08001927 def CASE11( self, main ):
1928 """
Jon Hallb1290e82014-11-18 16:17:48 -05001929 Switch Down
Jon Hall6aec96b2015-01-19 14:49:31 -08001930 """
1931 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05001932 import time
1933
Jon Hall8f89dda2015-01-22 16:03:33 -08001934 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hallb1290e82014-11-18 16:17:48 -05001935
1936 description = "Killing a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001937 main.log.report( description )
1938 main.case( description )
1939 switch = main.params[ 'kill' ][ 'switch' ]
1940 switchDPID = main.params[ 'kill' ][ 'dpid' ]
Jon Hallb1290e82014-11-18 16:17:48 -05001941
Jon Hall6aec96b2015-01-19 14:49:31 -08001942 # TODO: Make this switch parameterizable
1943 main.step( "Kill " + switch )
1944 main.log.report( "Deleting " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08001945 main.Mininet1.delSwitch( switch )
1946 main.log.info( "Waiting " + str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001947 " seconds for switch down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001948 time.sleep( switchSleep )
1949 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001950 # Peek at the deleted switch
1951 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001952 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001953 if device and device[ 'available' ] is False:
Jon Hall94fd0472014-12-08 11:52:42 -08001954 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001955 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08001956 onpass="Kill switch succesful",
1957 onfail="Failed to kill switch?" )
Jon Hallb1290e82014-11-18 16:17:48 -05001958
Jon Hall6aec96b2015-01-19 14:49:31 -08001959 def CASE12( self, main ):
1960 """
Jon Hallb1290e82014-11-18 16:17:48 -05001961 Switch Up
Jon Hall6aec96b2015-01-19 14:49:31 -08001962 """
1963 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05001964 import time
Jon Hall669173b2014-12-17 11:36:30 -08001965
Jon Hall8f89dda2015-01-22 16:03:33 -08001966 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall6aec96b2015-01-19 14:49:31 -08001967 switch = main.params[ 'kill' ][ 'switch' ]
1968 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1969 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallb1290e82014-11-18 16:17:48 -05001970 description = "Adding a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001971 main.log.report( description )
1972 main.case( description )
Jon Hallb1290e82014-11-18 16:17:48 -05001973
Jon Hall6aec96b2015-01-19 14:49:31 -08001974 main.step( "Add back " + switch )
1975 main.log.report( "Adding back " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08001976 main.Mininet1.addSwitch( switch, dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001977 # TODO: New dpid or same? Ask Thomas?
1978 for peer in links:
Jon Hall8f89dda2015-01-22 16:03:33 -08001979 main.Mininet1.addLink( switch, peer )
Jon Hall58c76b72015-02-23 11:09:24 -08001980 main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
1981 count=numControllers,
1982 ip1=ONOS1Ip,
1983 port1=ONOS1Port,
1984 ip2=ONOS2Ip,
1985 port2=ONOS2Port,
1986 ip3=ONOS3Ip,
1987 port3=ONOS3Port,
1988 ip4=ONOS4Ip,
1989 port4=ONOS4Port,
1990 ip5=ONOS5Ip,
1991 port5=ONOS5Port,
1992 ip6=ONOS6Ip,
1993 port6=ONOS6Port,
1994 ip7=ONOS7Ip,
1995 port7=ONOS7Port )
1996 main.log.info( "Waiting " + str( switchSleep ) +
1997 " seconds for switch up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001998 time.sleep( switchSleep )
1999 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002000 # Peek at the deleted switch
2001 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08002002 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002003 if device and device[ 'available' ]:
Jon Hall94fd0472014-12-08 11:52:42 -08002004 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002005 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08002006 onpass="add switch succesful",
2007 onfail="Failed to add switch?" )
Jon Hallb1290e82014-11-18 16:17:48 -05002008
Jon Hall6aec96b2015-01-19 14:49:31 -08002009 def CASE13( self, main ):
2010 """
Jon Hallb1290e82014-11-18 16:17:48 -05002011 Clean up
Jon Hall6aec96b2015-01-19 14:49:31 -08002012 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002013 import os
2014 import time
Jon Hall6aec96b2015-01-19 14:49:31 -08002015 # TODO: make use of this elsewhere
2016 ips = []
Jon Hall8f89dda2015-01-22 16:03:33 -08002017 ips.append( ONOS1Ip )
2018 ips.append( ONOS2Ip )
2019 ips.append( ONOS3Ip )
2020 ips.append( ONOS4Ip )
2021 ips.append( ONOS5Ip )
2022 ips.append( ONOS6Ip )
2023 ips.append( ONOS7Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -08002024
2025 # printing colors to terminal
Jon Hall669173b2014-12-17 11:36:30 -08002026 colors = {}
Jon Hall6aec96b2015-01-19 14:49:31 -08002027 colors[ 'cyan' ] = '\033[96m'
2028 colors[ 'purple' ] = '\033[95m'
2029 colors[ 'blue' ] = '\033[94m'
2030 colors[ 'green' ] = '\033[92m'
2031 colors[ 'yellow' ] = '\033[93m'
2032 colors[ 'red' ] = '\033[91m'
2033 colors[ 'end' ] = '\033[0m'
Jon Hall368769f2014-11-19 15:43:35 -08002034 description = "Test Cleanup"
Jon Hall6aec96b2015-01-19 14:49:31 -08002035 main.log.report( description )
2036 main.case( description )
2037 main.step( "Killing tcpdumps" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002038 main.Mininet2.stopTcpdump()
Jon Hallb1290e82014-11-18 16:17:48 -05002039
Jon Hall6aec96b2015-01-19 14:49:31 -08002040 main.step( "Checking ONOS Logs for errors" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002041 for i in range( 7 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002042 print colors[ 'purple' ] + "Checking logs for errors on " + \
Jon Hall8f89dda2015-01-22 16:03:33 -08002043 "ONOS" + str( i + 1 ) + ":" + colors[ 'end' ]
2044 print main.ONOSbench.checkLogs( ips[ i ] )
Jon Hall94fd0472014-12-08 11:52:42 -08002045
Jon Hall6aec96b2015-01-19 14:49:31 -08002046 main.step( "Copying MN pcap and ONOS log files to test station" )
Jon Hallb1290e82014-11-18 16:17:48 -05002047 testname = main.TEST
Jon Hall8f89dda2015-01-22 16:03:33 -08002048 teststationUser = main.params[ 'TESTONUSER' ]
2049 teststationIP = main.params[ 'TESTONIP' ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002050 # NOTE: MN Pcap file is being saved to ~/packet_captures
Jon Hall73cf9cc2014-11-20 22:28:38 -08002051 # scp this file as MN and TestON aren't necessarily the same vm
Jon Hall6aec96b2015-01-19 14:49:31 -08002052 # FIXME: scp
2053 # mn files
2054 # TODO: Load these from params
2055 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002056 logFolder = "/opt/onos/log/"
2057 logFiles = [ "karaf.log", "karaf.log.1" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002058 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002059 dstDir = "~/packet_captures/"
2060 for f in logFiles:
2061 for i in range( 7 ):
2062 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
2063 logFolder + f + " " +
2064 teststationUser + "@" +
2065 teststationIP + ":" +
2066 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002067 "-ONOS" + str( i + 1 ) + "-" +
2068 f )
Jon Hall58c76b72015-02-23 11:09:24 -08002069 main.ONOSbench.handle.expect( "\$" )
2070
Jon Hall6aec96b2015-01-19 14:49:31 -08002071 # std*.log's
2072 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002073 logFolder = "/opt/onos/var/"
2074 logFiles = [ "stderr.log", "stdout.log" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002075 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002076 dstDir = "~/packet_captures/"
2077 for f in logFiles:
2078 for i in range( 7 ):
2079 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
2080 logFolder + f + " " +
2081 teststationUser + "@" +
2082 teststationIP + ":" +
2083 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002084 "-ONOS" + str( i + 1 ) + "-" +
2085 f )
Jon Hall58c76b72015-02-23 11:09:24 -08002086 main.ONOSbench.handle.expect( "\$" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002087 # sleep so scp can finish
2088 time.sleep( 10 )
Jon Hall58c76b72015-02-23 11:09:24 -08002089 main.Mininet1.stopNet()
Jon Hall6aec96b2015-01-19 14:49:31 -08002090 main.step( "Packing and rotating pcap archives" )
2091 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002092
Jon Hall6aec96b2015-01-19 14:49:31 -08002093 # TODO: actually check something here
2094 utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08002095 onpass="Test cleanup successful",
2096 onfail="Test cleanup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002097
Jon Hall6aec96b2015-01-19 14:49:31 -08002098 def CASE14( self, main ):
2099 """
Jon Hall94fd0472014-12-08 11:52:42 -08002100 start election app on all onos nodes
Jon Hall6aec96b2015-01-19 14:49:31 -08002101 """
Jon Hall8f89dda2015-01-22 16:03:33 -08002102 leaderResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002103 # install app on onos 1
2104 main.log.info( "Install leadership election app" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002105 main.ONOScli1.featureInstall( "onos-app-election" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002106 # wait for election
2107 # check for leader
Jon Hall8f89dda2015-01-22 16:03:33 -08002108 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002109 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08002110 if leader == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08002111 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08002112 pass
Jon Hall6aec96b2015-01-19 14:49:31 -08002113 elif leader is None:
2114 # No leader elected
2115 main.log.report( "No leader was elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002116 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002117 elif leader == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002118 # error in response
2119 # TODO: add check for "Command not found:" in the driver, this
2120 # means the app isn't loaded
Jon Hall8f89dda2015-01-22 16:03:33 -08002121 main.log.report( "Something is wrong with electionTestLeader" +
Jon Hall6aec96b2015-01-19 14:49:31 -08002122 " function, check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002123 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002124 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002125 # error in response
2126 main.log.report(
Jon Hall8f89dda2015-01-22 16:03:33 -08002127 "Unexpected response from electionTestLeader function:'" +
2128 str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002129 "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002130 leaderResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08002131
Jon Hall6aec96b2015-01-19 14:49:31 -08002132 # install on other nodes and check for leader.
2133 # Should be onos1 and each app should show the same leader
Jon Hall8f89dda2015-01-22 16:03:33 -08002134 for controller in range( 2, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002135 # loop through ONOScli handlers
2136 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002137 node.featureInstall( "onos-app-election" )
2138 leaderN = node.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002139 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08002140 if leaderN == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08002141 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08002142 pass
2143 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002144 # error in response
2145 # TODO: add check for "Command not found:" in the driver, this
2146 # means the app isn't loaded
2147 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002148 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08002149 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002150 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002151 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08002152 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002153 main.log.report( "ONOS" + str( controller ) + " sees " +
2154 str( leaderN ) +
2155 " as the leader of the election app. Leader" +
2156 " should be " +
2157 str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002158 if leaderResult:
Jon Hall6aec96b2015-01-19 14:49:31 -08002159 main.log.report( "Leadership election tests passed( consistent " +
2160 "view of leader across listeners and a leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002161 "was elected )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002162 utilities.assert_equals(
2163 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002164 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002165 onpass="Leadership election passed",
2166 onfail="Something went wrong with Leadership election" )
Jon Hall94fd0472014-12-08 11:52:42 -08002167
Jon Hall6aec96b2015-01-19 14:49:31 -08002168 def CASE15( self, main ):
2169 """
Jon Hall669173b2014-12-17 11:36:30 -08002170 Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -08002171 """
Jon Hall8f89dda2015-01-22 16:03:33 -08002172 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002173 description = "Check that Leadership Election is still functional"
Jon Hall6aec96b2015-01-19 14:49:31 -08002174 main.log.report( description )
2175 main.case( description )
2176 main.step( "Find current leader and withdraw" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002177 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002178 # TODO: do some sanity checking on leader before using it
Jon Hall8f89dda2015-01-22 16:03:33 -08002179 withdrawResult = main.FALSE
2180 if leader == ONOS1Ip:
2181 oldLeader = getattr( main, "ONOScli1" )
2182 elif leader == ONOS2Ip:
2183 oldLeader = getattr( main, "ONOScli2" )
2184 elif leader == ONOS3Ip:
2185 oldLeader = getattr( main, "ONOScli3" )
2186 elif leader == ONOS4Ip:
2187 oldLeader = getattr( main, "ONOScli4" )
2188 elif leader == ONOS5Ip:
2189 oldLeader = getattr( main, "ONOScli5" )
2190 elif leader == ONOS6Ip:
2191 oldLeader = getattr( main, "ONOScli6" )
2192 elif leader == ONOS7Ip:
2193 oldLeader = getattr( main, "ONOScli7" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002194 elif leader is None or leader == main.FALSE:
2195 main.log.report(
2196 "Leader for the election app should be an ONOS node," +
Jon Hall58c76b72015-02-23 11:09:24 -08002197 "instead got '" + str( leader ) + "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002198 leaderResult = main.FALSE
Jon Hall63604932015-02-26 17:09:50 -08002199 oldLeader = None
2200 else:
2201 main.log.error( "Leader election --- why am I HERE?!?")
2202 if oldLeader:
2203 withdrawResult = oldLeader.electionTestWithdraw()
Jon Hall6aec96b2015-01-19 14:49:31 -08002204 utilities.assert_equals(
2205 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002206 actual=withdrawResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002207 onpass="App was withdrawn from election",
2208 onfail="App was not withdrawn from election" )
Jon Hall669173b2014-12-17 11:36:30 -08002209
Jon Hall6aec96b2015-01-19 14:49:31 -08002210 main.step( "Make sure new leader is elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002211 leaderList = []
2212 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002213 # loop through ONOScli handlers
2214 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002215 leaderList.append( node.electionTestLeader() )
2216 for leaderN in leaderList:
Jon Hall669173b2014-12-17 11:36:30 -08002217 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08002218 main.log.report(
Jon Hall58c76b72015-02-23 11:09:24 -08002219 "ONOS" + str( controller ) +
2220 " still sees " + str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002221 " as leader after they withdrew" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002222 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002223 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002224 # error in response
2225 # TODO: add check for "Command not found:" in the driver, this
2226 # means the app isn't loaded
2227 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002228 "electionTestLeader function, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002229 "check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002230 leaderResult = main.FALSE
2231 consistentLeader = main.FALSE
2232 if len( set( leaderList ) ) == 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08002233 main.log.info( "Each Election-app sees '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08002234 str( leaderList[ 0 ] ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002235 "' as the leader" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002236 consistentLeader = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002237 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002238 main.log.report(
2239 "Inconsistent responses for leader of Election-app:" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002240 for n in range( len( leaderList ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002241 main.log.report( "ONOS" + str( n + 1 ) + " response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002242 str( leaderList[ n ] ) )
2243 if leaderResult:
2244 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002245 "view of leader across listeners and a new " +
2246 "leader was elected when the old leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002247 "resigned )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002248 utilities.assert_equals(
2249 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002250 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002251 onpass="Leadership election passed",
2252 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002253
Jon Hall58c76b72015-02-23 11:09:24 -08002254 main.step( "Run for election on old leader( just so everyone " +
2255 "is in the hat )" )
Jon Hall63604932015-02-26 17:09:50 -08002256 if oldLeader:
2257 runResult = oldLeader.electionTestRun()
2258 else:
2259 runResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002260 utilities.assert_equals(
2261 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002262 actual=runResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002263 onpass="App re-ran for election",
2264 onfail="App failed to run for election" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002265 if consistentLeader == main.TRUE:
2266 afterRun = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002267 # verify leader didn't just change
Jon Hall8f89dda2015-01-22 16:03:33 -08002268 if afterRun == leaderList[ 0 ]:
2269 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002270 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08002271 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002272 # TODO: assert on run and withdraw results?
Jon Hall669173b2014-12-17 11:36:30 -08002273
Jon Hall6aec96b2015-01-19 14:49:31 -08002274 utilities.assert_equals(
2275 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002276 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002277 onpass="Leadership election passed",
2278 onfail="Something went wrong with Leadership election after " +
2279 "the old leader re-ran for election" )