blob: e9beb0cc507a92f6c39d5016cc865729ad2d3ec0 [file] [log] [blame]
Jon Hall6aec96b2015-01-19 14:49:31 -08001"""
Jon Hall73cf9cc2014-11-20 22:28:38 -08002Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign mastership to controllers
8CASE3: Assign intents
9CASE4: Ping across added host intents
10CASE5: Reading state of ONOS
11CASE6: The Failure case.
12CASE7: Check state after control plane failure
13CASE8: Compare topo
14CASE9: Link s3-s28 down
15CASE10: Link s3-s28 up
16CASE11: Switch down
17CASE12: Switch up
18CASE13: Clean up
Jon Hall669173b2014-12-17 11:36:30 -080019CASE14: start election app on all onos nodes
20CASE15: Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -080021"""
Jon Hall8f89dda2015-01-22 16:03:33 -080022
23
Jon Hall73cf9cc2014-11-20 22:28:38 -080024class HATestClusterRestart:
25
Jon Hall6aec96b2015-01-19 14:49:31 -080026 def __init__( self ):
Jon Hall73cf9cc2014-11-20 22:28:38 -080027 self.default = ''
28
Jon Hall6aec96b2015-01-19 14:49:31 -080029 def CASE1( self, main ):
30 """
Jon Hall73cf9cc2014-11-20 22:28:38 -080031 CASE1 is to compile ONOS and push it to the test machines
32
33 Startup sequence:
34 git pull
35 mvn clean install
36 onos-package
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-install -f
41 onos-wait-for-start
Jon Hall6aec96b2015-01-19 14:49:31 -080042 """
43 main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
44 "initialization" )
45 main.case( "Setting up test environment" )
46 # TODO: save all the timers and output them for plotting
Jon Hall73cf9cc2014-11-20 22:28:38 -080047
48 # load some vairables from the params file
Jon Hall8f89dda2015-01-22 16:03:33 -080049 PULLCODE = False
Jon Hall6aec96b2015-01-19 14:49:31 -080050 if main.params[ 'Git' ] == 'True':
Jon Hall8f89dda2015-01-22 16:03:33 -080051 PULLCODE = True
Jon Hall529a37f2015-01-28 10:02:00 -080052 gitBranch = main.params[ 'branch' ]
Jon Hall8f89dda2015-01-22 16:03:33 -080053 cellName = main.params[ 'ENV' ][ 'cellName' ]
Jon Hall6aec96b2015-01-19 14:49:31 -080054
55 # set global variables
Jon Hall8f89dda2015-01-22 16:03:33 -080056 global ONOS1Ip
57 global ONOS1Port
58 global ONOS2Ip
59 global ONOS2Port
60 global ONOS3Ip
61 global ONOS3Port
62 global ONOS4Ip
63 global ONOS4Port
64 global ONOS5Ip
65 global ONOS5Port
66 global ONOS6Ip
67 global ONOS6Port
68 global ONOS7Ip
69 global ONOS7Port
70 global numControllers
Jon Hall73cf9cc2014-11-20 22:28:38 -080071
Jon Hall8f89dda2015-01-22 16:03:33 -080072 ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
73 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
74 ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
75 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
76 ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
77 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
78 ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
79 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
80 ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
81 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
82 ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
83 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
84 ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
85 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
86 numControllers = int( main.params[ 'num_controllers' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -080087
Jon Hall6aec96b2015-01-19 14:49:31 -080088 main.step( "Applying cell variable to environment" )
Jon Hall8f89dda2015-01-22 16:03:33 -080089 cellResult = main.ONOSbench.setCell( cellName )
90 verifyResult = main.ONOSbench.verifyCell()
Jon Hall73cf9cc2014-11-20 22:28:38 -080091
Jon Hall6aec96b2015-01-19 14:49:31 -080092 # FIXME:this is short term fix
93 main.log.report( "Removing raft logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -080094 main.ONOSbench.onosRemoveRaftLogs()
Jon Hall6aec96b2015-01-19 14:49:31 -080095 main.log.report( "Uninstalling ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -080096 main.ONOSbench.onosUninstall( ONOS1Ip )
97 main.ONOSbench.onosUninstall( ONOS2Ip )
98 main.ONOSbench.onosUninstall( ONOS3Ip )
99 main.ONOSbench.onosUninstall( ONOS4Ip )
100 main.ONOSbench.onosUninstall( ONOS5Ip )
101 main.ONOSbench.onosUninstall( ONOS6Ip )
102 main.ONOSbench.onosUninstall( ONOS7Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800103
Jon Hall8f89dda2015-01-22 16:03:33 -0800104 cleanInstallResult = main.TRUE
105 gitPullResult = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800106
Jon Hall97f31752015-02-04 12:01:04 -0800107 main.step( "Starting Mininet" )
108 main.Mininet1.startNet( )
109
Jon Hall6aec96b2015-01-19 14:49:31 -0800110 main.step( "Compiling the latest version of ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800111 if PULLCODE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800112 # TODO Configure branch in params
113 main.step( "Git checkout and pull master" )
Jon Hall529a37f2015-01-28 10:02:00 -0800114 main.ONOSbench.gitCheckout( gitBranch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800115 gitPullResult = main.ONOSbench.gitPull()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800116
Jon Hall6aec96b2015-01-19 14:49:31 -0800117 main.step( "Using mvn clean & install" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800118 cleanInstallResult = main.ONOSbench.cleanInstall()
Jon Hall6aec96b2015-01-19 14:49:31 -0800119 else:
120 main.log.warn( "Did not pull new code so skipping mvn " +
121 "clean install" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800122 main.ONOSbench.getVersion( report=True )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800123
Jon Hall6aec96b2015-01-19 14:49:31 -0800124 main.step( "Creating ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800125 packageResult = main.ONOSbench.onosPackage()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800126
Jon Hall6aec96b2015-01-19 14:49:31 -0800127 main.step( "Installing ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800128 onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
129 node=ONOS1Ip )
130 onos2InstallResult = main.ONOSbench.onosInstall( options="-f",
131 node=ONOS2Ip )
132 onos3InstallResult = main.ONOSbench.onosInstall( options="-f",
133 node=ONOS3Ip )
134 onos4InstallResult = main.ONOSbench.onosInstall( options="-f",
135 node=ONOS4Ip )
136 onos5InstallResult = main.ONOSbench.onosInstall( options="-f",
137 node=ONOS5Ip )
138 onos6InstallResult = main.ONOSbench.onosInstall( options="-f",
139 node=ONOS6Ip )
140 onos7InstallResult = main.ONOSbench.onosInstall( options="-f",
141 node=ONOS7Ip )
142 onosInstallResult = onos1InstallResult and onos2InstallResult\
143 and onos3InstallResult and onos4InstallResult\
144 and onos5InstallResult and onos6InstallResult\
145 and onos7InstallResult
Jon Hall73cf9cc2014-11-20 22:28:38 -0800146
Jon Hall6aec96b2015-01-19 14:49:31 -0800147 main.step( "Checking if ONOS is up yet" )
148 # TODO check bundle:list?
149 for i in range( 2 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800150 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
151 if not onos1Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800152 main.log.report( "ONOS1 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800153 main.ONOSbench.onosStop( ONOS1Ip )
154 main.ONOSbench.onosStart( ONOS1Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800155 onos2Isup = main.ONOSbench.isup( ONOS2Ip )
156 if not onos2Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800157 main.log.report( "ONOS2 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800158 main.ONOSbench.onosStop( ONOS2Ip )
159 main.ONOSbench.onosStart( ONOS2Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800160 onos3Isup = main.ONOSbench.isup( ONOS3Ip )
161 if not onos3Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800162 main.log.report( "ONOS3 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800163 main.ONOSbench.onosStop( ONOS3Ip )
164 main.ONOSbench.onosStart( ONOS3Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800165 onos4Isup = main.ONOSbench.isup( ONOS4Ip )
166 if not onos4Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800167 main.log.report( "ONOS4 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800168 main.ONOSbench.onosStop( ONOS4Ip )
169 main.ONOSbench.onosStart( ONOS4Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800170 onos5Isup = main.ONOSbench.isup( ONOS5Ip )
171 if not onos5Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800172 main.log.report( "ONOS5 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800173 main.ONOSbench.onosStop( ONOS5Ip )
174 main.ONOSbench.onosStart( ONOS5Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800175 onos6Isup = main.ONOSbench.isup( ONOS6Ip )
176 if not onos6Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800177 main.log.report( "ONOS6 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800178 main.ONOSbench.onosStop( ONOS6Ip )
179 main.ONOSbench.onosStart( ONOS6Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800180 onos7Isup = main.ONOSbench.isup( ONOS7Ip )
181 if not onos7Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800182 main.log.report( "ONOS7 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800183 main.ONOSbench.onosStop( ONOS7Ip )
184 main.ONOSbench.onosStart( ONOS7Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800185 onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
186 and onos4Isup and onos5Isup and onos6Isup and onos7Isup
187 if onosIsupResult == main.TRUE:
Jon Hall94fd0472014-12-08 11:52:42 -0800188 break
Jon Hall73cf9cc2014-11-20 22:28:38 -0800189
Jon Hall8f89dda2015-01-22 16:03:33 -0800190 cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
191 cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
192 cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
193 cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
194 cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
195 cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
196 cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
197 cliResults = cliResult1 and cliResult2 and cliResult3 and\
198 cliResult4 and cliResult5 and cliResult6 and cliResult7
Jon Hall73cf9cc2014-11-20 22:28:38 -0800199
Jon Hall6aec96b2015-01-19 14:49:31 -0800200 main.step( "Start Packet Capture MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800201 main.Mininet2.startTcpdump(
Jon Hall6aec96b2015-01-19 14:49:31 -0800202 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
203 + "-MN.pcap",
204 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
205 port=main.params[ 'MNtcpdump' ][ 'port' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800206
Jon Hall8f89dda2015-01-22 16:03:33 -0800207 case1Result = ( cleanInstallResult and packageResult and
208 cellResult and verifyResult and onosInstallResult
209 and onosIsupResult and cliResults )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800210
Jon Hall8f89dda2015-01-22 16:03:33 -0800211 utilities.assert_equals( expect=main.TRUE, actual=case1Result,
212 onpass="Test startup successful",
213 onfail="Test startup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800214
Jon Hall8f89dda2015-01-22 16:03:33 -0800215 if case1Result == main.FALSE:
Jon Hall94fd0472014-12-08 11:52:42 -0800216 main.cleanup()
217 main.exit()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800218
Jon Hall6aec96b2015-01-19 14:49:31 -0800219 def CASE2( self, main ):
220 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800221 Assign mastership to controllers
Jon Hall6aec96b2015-01-19 14:49:31 -0800222 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800223 import re
224
Jon Hall6aec96b2015-01-19 14:49:31 -0800225 main.log.report( "Assigning switches to controllers" )
226 main.case( "Assigning Controllers" )
227 main.step( "Assign switches to controllers" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800228
Jon Hall6aec96b2015-01-19 14:49:31 -0800229 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800230 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -0800231 sw=str( i ),
Jon Hall8f89dda2015-01-22 16:03:33 -0800232 count=numControllers,
233 ip1=ONOS1Ip, port1=ONOS1Port,
234 ip2=ONOS2Ip, port2=ONOS2Port,
235 ip3=ONOS3Ip, port3=ONOS3Port,
236 ip4=ONOS4Ip, port4=ONOS4Port,
237 ip5=ONOS5Ip, port5=ONOS5Port,
238 ip6=ONOS6Ip, port6=ONOS6Port,
239 ip7=ONOS7Ip, port7=ONOS7Port )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800240
Jon Hall8f89dda2015-01-22 16:03:33 -0800241 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800242 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800243 response = main.Mininet1.getSwController( "s" + str( i ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800244 try:
Jon Hall6aec96b2015-01-19 14:49:31 -0800245 main.log.info( str( response ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800246 except:
Jon Hall6aec96b2015-01-19 14:49:31 -0800247 main.log.info( repr( response ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800248 if re.search( "tcp:" + ONOS1Ip, response )\
249 and re.search( "tcp:" + ONOS2Ip, response )\
250 and re.search( "tcp:" + ONOS3Ip, response )\
251 and re.search( "tcp:" + ONOS4Ip, response )\
252 and re.search( "tcp:" + ONOS5Ip, response )\
253 and re.search( "tcp:" + ONOS6Ip, response )\
254 and re.search( "tcp:" + ONOS7Ip, response ):
255 mastershipCheck = mastershipCheck and main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800256 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800257 mastershipCheck = main.FALSE
258 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800259 main.log.report( "Switch mastership assigned correctly" )
260 utilities.assert_equals(
261 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800262 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800263 onpass="Switch mastership assigned correctly",
264 onfail="Switches not assigned correctly to controllers" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800265
Jon Hall6aec96b2015-01-19 14:49:31 -0800266 # Manually assign mastership to the controller we want
Jon Hall8f89dda2015-01-22 16:03:33 -0800267 roleCall = main.TRUE
268 roleCheck = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800269
Jon Hall6aec96b2015-01-19 14:49:31 -0800270 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800271 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
272 roleCall = roleCall and main.ONOScli1.deviceRole(
273 deviceId,
274 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800275 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800276 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
277 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800278 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800279 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800280
Jon Hall6aec96b2015-01-19 14:49:31 -0800281 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800282 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
283 roleCall = roleCall and main.ONOScli1.deviceRole(
284 deviceId,
285 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800286 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800287 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
288 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800289 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800290 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800291
Jon Hall6aec96b2015-01-19 14:49:31 -0800292 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800293 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
294 roleCall = roleCall and main.ONOScli1.deviceRole(
295 deviceId,
296 ONOS2Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800297 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800298 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
299 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800300 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800301 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800302
Jon Hall6aec96b2015-01-19 14:49:31 -0800303 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800304 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
305 roleCall = roleCall and main.ONOScli1.deviceRole(
306 deviceId,
307 ONOS2Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800308 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800309 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
310 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800311 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800312 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800313
Jon Hall6aec96b2015-01-19 14:49:31 -0800314 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800315 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
316 roleCall = roleCall and main.ONOScli1.deviceRole(
317 deviceId,
318 ONOS3Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800319 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800320 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
321 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800322 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800323 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800324
Jon Hall6aec96b2015-01-19 14:49:31 -0800325 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800326 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
327 roleCall = roleCall and main.ONOScli1.deviceRole(
328 deviceId,
329 ONOS3Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800330 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800331 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
332 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800333 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800334 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800335
Jon Hall6aec96b2015-01-19 14:49:31 -0800336 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800337 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
338 roleCall = roleCall and main.ONOScli1.deviceRole(
339 deviceId,
340 ONOS4Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800341 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800342 if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
343 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800344 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800345 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800346
Jon Hall6aec96b2015-01-19 14:49:31 -0800347 for i in range( 8, 18 ):
348 dpid = '3' + str( i ).zfill( 3 )
Jon Hall8f89dda2015-01-22 16:03:33 -0800349 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
350 roleCall = roleCall and main.ONOScli1.deviceRole(
351 deviceId,
352 ONOS5Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800353 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800354 if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
355 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800356 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800357 roleCheck = roleCheck and main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800358
Jon Hall8f89dda2015-01-22 16:03:33 -0800359 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
360 roleCall = roleCall and main.ONOScli1.deviceRole(
361 deviceId,
362 ONOS6Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800363 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800364 if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
365 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800366 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800367 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800368
Jon Hall6aec96b2015-01-19 14:49:31 -0800369 for i in range( 18, 28 ):
370 dpid = '6' + str( i ).zfill( 3 )
Jon Hall8f89dda2015-01-22 16:03:33 -0800371 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
372 roleCall = roleCall and main.ONOScli1.deviceRole(
373 deviceId,
374 ONOS7Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800375 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800376 if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
377 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800378 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800379 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800380
Jon Hall6aec96b2015-01-19 14:49:31 -0800381 utilities.assert_equals(
382 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800383 actual=roleCall,
Jon Hall6aec96b2015-01-19 14:49:31 -0800384 onpass="Re-assigned switch mastership to designated controller",
Jon Hall8f89dda2015-01-22 16:03:33 -0800385 onfail="Something wrong with deviceRole calls" )
Jon Hall94fd0472014-12-08 11:52:42 -0800386
Jon Hall6aec96b2015-01-19 14:49:31 -0800387 utilities.assert_equals(
388 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800389 actual=roleCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800390 onpass="Switches were successfully reassigned to designated " +
391 "controller",
392 onfail="Switches were not successfully reassigned" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800393 mastershipCheck = mastershipCheck and roleCall and roleCheck
394 utilities.assert_equals( expect=main.TRUE, actual=mastershipCheck,
Jon Hall21270ac2015-02-16 17:59:55 -0800395 onpass="Switch mastership correctly assigned",
396 onfail="Error in (re)assigning switch" +
397 " mastership" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800398
Jon Hall6aec96b2015-01-19 14:49:31 -0800399 def CASE3( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800400 """
401 Assign intents
Jon Hall73cf9cc2014-11-20 22:28:38 -0800402 """
Jon Hall6aec96b2015-01-19 14:49:31 -0800403 # FIXME: we must reinstall intents until we have a persistant
404 # datastore!
Jon Hall73cf9cc2014-11-20 22:28:38 -0800405 import time
Jon Hall6aec96b2015-01-19 14:49:31 -0800406 main.log.report( "Adding host intents" )
407 main.case( "Adding host Intents" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800408
Jon Hall8f89dda2015-01-22 16:03:33 -0800409 main.step( "Discovering Hosts( Via pingall for now )" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800410 # FIXME: Once we have a host discovery mechanism, use that instead
Jon Hall73cf9cc2014-11-20 22:28:38 -0800411
Jon Hall6aec96b2015-01-19 14:49:31 -0800412 # install onos-app-fwd
413 main.log.info( "Install reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800414 main.ONOScli1.featureInstall( "onos-app-fwd" )
415 main.ONOScli2.featureInstall( "onos-app-fwd" )
416 main.ONOScli3.featureInstall( "onos-app-fwd" )
417 main.ONOScli4.featureInstall( "onos-app-fwd" )
418 main.ONOScli5.featureInstall( "onos-app-fwd" )
419 main.ONOScli6.featureInstall( "onos-app-fwd" )
420 main.ONOScli7.featureInstall( "onos-app-fwd" )
Jon Hall94fd0472014-12-08 11:52:42 -0800421
Jon Hall6aec96b2015-01-19 14:49:31 -0800422 # REACTIVE FWD test
Jon Hall8f89dda2015-01-22 16:03:33 -0800423 pingResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800424 time1 = time.time()
Jon Hall8f89dda2015-01-22 16:03:33 -0800425 pingResult = main.Mininet1.pingall()
Jon Hall529a37f2015-01-28 10:02:00 -0800426 utilities.assert_equals(
427 expect=main.TRUE,
428 actual=pingResult,
429 onpass="Reactive Pingall test passed",
430 onfail="Reactive Pingall failed, one or more ping pairs failed" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800431 time2 = time.time()
Jon Hall6aec96b2015-01-19 14:49:31 -0800432 main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800433
Jon Hall6aec96b2015-01-19 14:49:31 -0800434 # uninstall onos-app-fwd
435 main.log.info( "Uninstall reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800436 main.ONOScli1.featureUninstall( "onos-app-fwd" )
437 main.ONOScli2.featureUninstall( "onos-app-fwd" )
438 main.ONOScli3.featureUninstall( "onos-app-fwd" )
439 main.ONOScli4.featureUninstall( "onos-app-fwd" )
440 main.ONOScli5.featureUninstall( "onos-app-fwd" )
441 main.ONOScli6.featureUninstall( "onos-app-fwd" )
442 main.ONOScli7.featureUninstall( "onos-app-fwd" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800443 # timeout for fwd flows
444 time.sleep( 10 )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800445
Jon Hall6aec96b2015-01-19 14:49:31 -0800446 main.step( "Add host intents" )
447 # TODO: move the host numbers to params
Jon Hall8f89dda2015-01-22 16:03:33 -0800448 intentAddResult = True
Jon Hall6aec96b2015-01-19 14:49:31 -0800449 for i in range( 8, 18 ):
450 main.log.info( "Adding host intent between h" + str( i ) +
451 " and h" + str( i + 10 ) )
452 host1 = "00:00:00:00:00:" + \
453 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
454 host2 = "00:00:00:00:00:" + \
455 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800456 # NOTE: getHost can return None
457 host1Dict = main.ONOScli1.getHost( host1 )
458 host2Dict = main.ONOScli1.getHost( host2 )
459 host1Id = None
460 host2Id = None
461 if host1Dict and host2Dict:
462 host1Id = host1Dict.get( 'id', None )
463 host2Id = host2Dict.get( 'id', None )
Jon Hall8f89dda2015-01-22 16:03:33 -0800464 if host1Id and host2Id:
Jon Hall21270ac2015-02-16 17:59:55 -0800465
Jon Hall8f89dda2015-01-22 16:03:33 -0800466 tmpResult = main.ONOScli1.addHostIntent(
467 host1Id,
468 host2Id )
Jon Hall669173b2014-12-17 11:36:30 -0800469 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800470 main.log.error( "Error, getHost() failed" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800471 main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
472 sort_keys=True,
473 indent=4,
474 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800475 tmpResult = main.FALSE
476 intentAddResult = bool( pingResult and intentAddResult
477 and tmpResult )
Jon Hall529a37f2015-01-28 10:02:00 -0800478 # TODO Check that intents were added?
Jon Hall1b8f54a2015-02-04 13:24:20 -0800479 # Print the intent states
480 intents = main.ONOScli1.intents( )
481 intentStates = []
482 for intent in json.loads( intents ): # Iter through intents of a node
483 intentStates.append( intent.get( 'state', None ) )
484 out = [ (i, intentStates.count( i ) ) for i in set( intentStates ) ]
485 main.log.info( dict( out ) )
486
Jon Hall6aec96b2015-01-19 14:49:31 -0800487 utilities.assert_equals(
488 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800489 actual=intentAddResult,
Jon Hall529a37f2015-01-28 10:02:00 -0800490 onpass="Pushed host intents to ONOS",
491 onfail="Error in pushing host intents to ONOS" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800492 # TODO Check if intents all exist in datastore
Jon Hall73cf9cc2014-11-20 22:28:38 -0800493
Jon Hall6aec96b2015-01-19 14:49:31 -0800494 def CASE4( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800495 """
496 Ping across added host intents
497 """
498 description = " Ping across added host intents"
Jon Hall6aec96b2015-01-19 14:49:31 -0800499 main.log.report( description )
500 main.case( description )
Jon Hall8f89dda2015-01-22 16:03:33 -0800501 PingResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800502 for i in range( 8, 18 ):
503 ping = main.Mininet1.pingHost(
504 src="h" + str( i ), target="h" + str( i + 10 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800505 PingResult = PingResult and ping
Jon Hall6aec96b2015-01-19 14:49:31 -0800506 if ping == main.FALSE:
507 main.log.warn( "Ping failed between h" + str( i ) +
508 " and h" + str( i + 10 ) )
509 elif ping == main.TRUE:
510 main.log.info( "Ping test passed!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800511 # Don't set PingResult or you'd override failures
Jon Hall8f89dda2015-01-22 16:03:33 -0800512 if PingResult == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800513 main.log.report(
514 "Intents have not been installed correctly, pings failed." )
Jon Hall529a37f2015-01-28 10:02:00 -0800515 #TODO: pretty print
516 main.log.warn( "ONSO1 intents: " )
517 main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
518 sort_keys=True,
519 indent=4,
520 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800521 if PingResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800522 main.log.report(
523 "Intents have been installed correctly and verified by pings" )
524 utilities.assert_equals(
525 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800526 actual=PingResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800527 onpass="Intents have been installed correctly and pings work",
528 onfail="Intents have not been installed correctly, pings failed." )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800529
Jon Hall6aec96b2015-01-19 14:49:31 -0800530 def CASE5( self, main ):
531 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800532 Reading state of ONOS
Jon Hall6aec96b2015-01-19 14:49:31 -0800533 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800534 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800535 # assumes that sts is already in you PYTHONPATH
536 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -0800537
Jon Hall6aec96b2015-01-19 14:49:31 -0800538 main.log.report( "Setting up and gathering data for current state" )
539 main.case( "Setting up and gathering data for current state" )
540 # The general idea for this test case is to pull the state of
541 # ( intents,flows, topology,... ) from each ONOS node
542 # We can then compare them with eachother and also with past states
Jon Hall73cf9cc2014-11-20 22:28:38 -0800543
Jon Hall6aec96b2015-01-19 14:49:31 -0800544 main.step( "Get the Mastership of each switch from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800545 global mastershipState
546 mastershipState = []
Jon Hall94fd0472014-12-08 11:52:42 -0800547
Jon Hall6aec96b2015-01-19 14:49:31 -0800548 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -0800549 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
550 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
551 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
552 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
553 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
554 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
555 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
556 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
557 ONOS3MasterNotNull and ONOS4MasterNotNull and\
558 ONOS5MasterNotNull and ONOS6MasterNotNull and\
559 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -0800560 utilities.assert_equals(
561 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800562 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -0800563 onpass="Each device has a master",
564 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -0800565
Jon Hall8f89dda2015-01-22 16:03:33 -0800566 ONOS1Mastership = main.ONOScli1.roles()
567 ONOS2Mastership = main.ONOScli2.roles()
568 ONOS3Mastership = main.ONOScli3.roles()
569 ONOS4Mastership = main.ONOScli4.roles()
570 ONOS5Mastership = main.ONOScli5.roles()
571 ONOS6Mastership = main.ONOScli6.roles()
572 ONOS7Mastership = main.ONOScli7.roles()
573 if "Error" in ONOS1Mastership or not ONOS1Mastership\
574 or "Error" in ONOS2Mastership or not ONOS2Mastership\
575 or "Error" in ONOS3Mastership or not ONOS3Mastership\
576 or "Error" in ONOS4Mastership or not ONOS4Mastership\
577 or "Error" in ONOS5Mastership or not ONOS5Mastership\
578 or "Error" in ONOS6Mastership or not ONOS6Mastership\
579 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -0800580 main.log.report( "Error in getting ONOS roles" )
581 main.log.warn(
582 "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800583 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800584 main.log.warn(
585 "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800586 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800587 main.log.warn(
588 "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800589 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800590 main.log.warn(
591 "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800592 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800593 main.log.warn(
594 "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800595 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800596 main.log.warn(
597 "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800598 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800599 main.log.warn(
600 "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800601 repr( ONOS7Mastership ) )
602 consistentMastership = main.FALSE
603 elif ONOS1Mastership == ONOS2Mastership\
604 and ONOS1Mastership == ONOS3Mastership\
605 and ONOS1Mastership == ONOS4Mastership\
606 and ONOS1Mastership == ONOS5Mastership\
607 and ONOS1Mastership == ONOS6Mastership\
608 and ONOS1Mastership == ONOS7Mastership:
609 mastershipState = ONOS1Mastership
610 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800611 main.log.report(
612 "Switch roles are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800613 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800614 main.log.warn(
615 "ONOS1 roles: ",
616 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800617 json.loads( ONOS1Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800618 sort_keys=True,
619 indent=4,
620 separators=(
621 ',',
622 ': ' ) ) )
623 main.log.warn(
624 "ONOS2 roles: ",
625 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800626 json.loads( ONOS2Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800627 sort_keys=True,
628 indent=4,
629 separators=(
630 ',',
631 ': ' ) ) )
632 main.log.warn(
633 "ONOS3 roles: ",
634 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800635 json.loads( ONOS3Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800636 sort_keys=True,
637 indent=4,
638 separators=(
639 ',',
640 ': ' ) ) )
641 main.log.warn(
642 "ONOS4 roles: ",
643 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800644 json.loads( ONOS4Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800645 sort_keys=True,
646 indent=4,
647 separators=(
648 ',',
649 ': ' ) ) )
650 main.log.warn(
651 "ONOS5 roles: ",
652 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800653 json.loads( ONOS5Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800654 sort_keys=True,
655 indent=4,
656 separators=(
657 ',',
658 ': ' ) ) )
659 main.log.warn(
660 "ONOS6 roles: ",
661 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800662 json.loads( ONOS6Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800663 sort_keys=True,
664 indent=4,
665 separators=(
666 ',',
667 ': ' ) ) )
668 main.log.warn(
669 "ONOS7 roles: ",
670 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800671 json.loads( ONOS7Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800672 sort_keys=True,
673 indent=4,
674 separators=(
675 ',',
676 ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800677 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800678 utilities.assert_equals(
679 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800680 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -0800681 onpass="Switch roles are consistent across all ONOS nodes",
682 onfail="ONOS nodes have different views of switch roles" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800683
Jon Hall6aec96b2015-01-19 14:49:31 -0800684 main.step( "Get the intents from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800685 global intentState
686 intentState = []
687 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
688 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
689 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
690 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
691 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
692 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
693 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
694 intentCheck = main.FALSE
695 if "Error" in ONOS1Intents or not ONOS1Intents\
696 or "Error" in ONOS2Intents or not ONOS2Intents\
697 or "Error" in ONOS3Intents or not ONOS3Intents\
698 or "Error" in ONOS4Intents or not ONOS4Intents\
699 or "Error" in ONOS5Intents or not ONOS5Intents\
700 or "Error" in ONOS6Intents or not ONOS6Intents\
701 or "Error" in ONOS7Intents or not ONOS7Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -0800702 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800703 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
704 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
705 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
706 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
707 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
708 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
709 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
710 elif ONOS1Intents == ONOS2Intents\
711 and ONOS1Intents == ONOS3Intents\
712 and ONOS1Intents == ONOS4Intents\
713 and ONOS1Intents == ONOS5Intents\
714 and ONOS1Intents == ONOS6Intents\
715 and ONOS1Intents == ONOS7Intents:
716 intentState = ONOS1Intents
717 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800718 main.log.report( "Intents are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800719 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800720 main.log.warn(
721 "ONOS1 intents: ",
722 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800723 json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800724 sort_keys=True,
725 indent=4,
726 separators=(
727 ',',
728 ': ' ) ) )
729 main.log.warn(
730 "ONOS2 intents: ",
731 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800732 json.loads( ONOS2Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800733 sort_keys=True,
734 indent=4,
735 separators=(
736 ',',
737 ': ' ) ) )
738 main.log.warn(
739 "ONOS3 intents: ",
740 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800741 json.loads( ONOS3Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800742 sort_keys=True,
743 indent=4,
744 separators=(
745 ',',
746 ': ' ) ) )
747 main.log.warn(
748 "ONOS4 intents: ",
749 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800750 json.loads( ONOS4Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800751 sort_keys=True,
752 indent=4,
753 separators=(
754 ',',
755 ': ' ) ) )
756 main.log.warn(
757 "ONOS5 intents: ",
758 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800759 json.loads( ONOS5Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800760 sort_keys=True,
761 indent=4,
762 separators=(
763 ',',
764 ': ' ) ) )
765 main.log.warn(
766 "ONOS6 intents: ",
767 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800768 json.loads( ONOS6Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800769 sort_keys=True,
770 indent=4,
771 separators=(
772 ',',
773 ': ' ) ) )
774 main.log.warn(
775 "ONOS7 intents: ",
776 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800777 json.loads( ONOS7Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800778 sort_keys=True,
779 indent=4,
780 separators=(
781 ',',
782 ': ' ) ) )
783 utilities.assert_equals(
784 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800785 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800786 onpass="Intents are consistent across all ONOS nodes",
787 onfail="ONOS nodes have different views of intents" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800788
Jon Hall6aec96b2015-01-19 14:49:31 -0800789 main.step( "Get the flows from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800790 global flowState
791 flowState = []
792 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
793 ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
794 ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
795 ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
796 ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
797 ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
798 ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
799 ONOS1FlowsJson = json.loads( ONOS1Flows )
800 ONOS2FlowsJson = json.loads( ONOS2Flows )
801 ONOS3FlowsJson = json.loads( ONOS3Flows )
802 ONOS4FlowsJson = json.loads( ONOS4Flows )
803 ONOS5FlowsJson = json.loads( ONOS5Flows )
804 ONOS6FlowsJson = json.loads( ONOS6Flows )
805 ONOS7FlowsJson = json.loads( ONOS7Flows )
806 flowCheck = main.FALSE
807 if "Error" in ONOS1Flows or not ONOS1Flows\
808 or "Error" in ONOS2Flows or not ONOS2Flows\
809 or "Error" in ONOS3Flows or not ONOS3Flows\
810 or "Error" in ONOS4Flows or not ONOS4Flows\
811 or "Error" in ONOS5Flows or not ONOS5Flows\
812 or "Error" in ONOS6Flows or not ONOS6Flows\
813 or "Error" in ONOS7Flows or not ONOS7Flows:
Jon Hall6aec96b2015-01-19 14:49:31 -0800814 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800815 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
816 main.log.warn( "ONOS2 flows repsponse: " + ONOS2Flows )
817 main.log.warn( "ONOS3 flows repsponse: " + ONOS3Flows )
818 main.log.warn( "ONOS4 flows repsponse: " + ONOS4Flows )
819 main.log.warn( "ONOS5 flows repsponse: " + ONOS5Flows )
820 main.log.warn( "ONOS6 flows repsponse: " + ONOS6Flows )
821 main.log.warn( "ONOS7 flows repsponse: " + ONOS7Flows )
822 elif len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
823 and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
824 and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
825 and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
826 and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
827 and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
Jon Hall6aec96b2015-01-19 14:49:31 -0800828 # TODO: Do a better check, maybe compare flows on switches?
Jon Hall8f89dda2015-01-22 16:03:33 -0800829 flowState = ONOS1Flows
830 flowCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800831 main.log.report( "Flow count is consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800832 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800833 main.log.warn( "ONOS1 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800834 json.dumps( ONOS1FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800835 indent=4, separators=( ',', ': ' ) ) )
836 main.log.warn( "ONOS2 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800837 json.dumps( ONOS2FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800838 indent=4, separators=( ',', ': ' ) ) )
839 main.log.warn( "ONOS3 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800840 json.dumps( ONOS3FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800841 indent=4, separators=( ',', ': ' ) ) )
842 main.log.warn( "ONOS4 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800843 json.dumps( ONOS4FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800844 indent=4, separators=( ',', ': ' ) ) )
845 main.log.warn( "ONOS5 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800846 json.dumps( ONOS5FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800847 indent=4, separators=( ',', ': ' ) ) )
848 main.log.warn( "ONOS6 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800849 json.dumps( ONOS6FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800850 indent=4, separators=( ',', ': ' ) ) )
851 main.log.warn( "ONOS7 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800852 json.dumps( ONOS7FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800853 indent=4, separators=( ',', ': ' ) ) )
854 utilities.assert_equals(
855 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800856 actual=flowCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800857 onpass="The flow count is consistent across all ONOS nodes",
858 onfail="ONOS nodes have different flow counts" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800859
Jon Hall6aec96b2015-01-19 14:49:31 -0800860 main.step( "Get the OF Table entries" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800861 global flows
Jon Hall6aec96b2015-01-19 14:49:31 -0800862 flows = []
863 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800864 flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800865
Jon Hall6aec96b2015-01-19 14:49:31 -0800866 # TODO: Compare switch flow tables with ONOS flow tables
Jon Hall73cf9cc2014-11-20 22:28:38 -0800867
Jon Hall6aec96b2015-01-19 14:49:31 -0800868 main.step( "Start continuous pings" )
869 main.Mininet2.pingLong(
870 src=main.params[ 'PING' ][ 'source1' ],
871 target=main.params[ 'PING' ][ 'target1' ],
872 pingTime=500 )
873 main.Mininet2.pingLong(
874 src=main.params[ 'PING' ][ 'source2' ],
875 target=main.params[ 'PING' ][ 'target2' ],
876 pingTime=500 )
877 main.Mininet2.pingLong(
878 src=main.params[ 'PING' ][ 'source3' ],
879 target=main.params[ 'PING' ][ 'target3' ],
880 pingTime=500 )
881 main.Mininet2.pingLong(
882 src=main.params[ 'PING' ][ 'source4' ],
883 target=main.params[ 'PING' ][ 'target4' ],
884 pingTime=500 )
885 main.Mininet2.pingLong(
886 src=main.params[ 'PING' ][ 'source5' ],
887 target=main.params[ 'PING' ][ 'target5' ],
888 pingTime=500 )
889 main.Mininet2.pingLong(
890 src=main.params[ 'PING' ][ 'source6' ],
891 target=main.params[ 'PING' ][ 'target6' ],
892 pingTime=500 )
893 main.Mininet2.pingLong(
894 src=main.params[ 'PING' ][ 'source7' ],
895 target=main.params[ 'PING' ][ 'target7' ],
896 pingTime=500 )
897 main.Mininet2.pingLong(
898 src=main.params[ 'PING' ][ 'source8' ],
899 target=main.params[ 'PING' ][ 'target8' ],
900 pingTime=500 )
901 main.Mininet2.pingLong(
902 src=main.params[ 'PING' ][ 'source9' ],
903 target=main.params[ 'PING' ][ 'target9' ],
904 pingTime=500 )
905 main.Mininet2.pingLong(
906 src=main.params[ 'PING' ][ 'source10' ],
907 target=main.params[ 'PING' ][ 'target10' ],
908 pingTime=500 )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800909
Jon Hall6aec96b2015-01-19 14:49:31 -0800910 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800911 ctrls = []
912 count = 1
913 while True:
914 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -0800915 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
916 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
917 temp = temp + ( "ONOS" + str( count ), )
918 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
919 temp = temp + \
920 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
921 ctrls.append( temp )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800922 count = count + 1
923 else:
924 break
Jon Hall6aec96b2015-01-19 14:49:31 -0800925 MNTopo = TestONTopology(
926 main.Mininet1,
927 ctrls ) # can also add Intent API info for intent operations
Jon Hall73cf9cc2014-11-20 22:28:38 -0800928
Jon Hall6aec96b2015-01-19 14:49:31 -0800929 main.step( "Collecting topology information from ONOS" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800930 devices = []
931 devices.append( main.ONOScli1.devices() )
932 devices.append( main.ONOScli2.devices() )
933 devices.append( main.ONOScli3.devices() )
934 devices.append( main.ONOScli4.devices() )
935 devices.append( main.ONOScli5.devices() )
936 devices.append( main.ONOScli6.devices() )
937 devices.append( main.ONOScli7.devices() )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800938 hosts = []
939 hosts.append( main.ONOScli1.hosts() )
940 hosts.append( main.ONOScli2.hosts() )
941 hosts.append( main.ONOScli3.hosts() )
942 hosts.append( main.ONOScli4.hosts() )
943 hosts.append( main.ONOScli5.hosts() )
944 hosts.append( main.ONOScli6.hosts() )
945 hosts.append( main.ONOScli7.hosts() )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800946 ports = []
947 ports.append( main.ONOScli1.ports() )
948 ports.append( main.ONOScli2.ports() )
949 ports.append( main.ONOScli3.ports() )
950 ports.append( main.ONOScli4.ports() )
951 ports.append( main.ONOScli5.ports() )
952 ports.append( main.ONOScli6.ports() )
953 ports.append( main.ONOScli7.ports() )
954 links = []
955 links.append( main.ONOScli1.links() )
956 links.append( main.ONOScli2.links() )
957 links.append( main.ONOScli3.links() )
958 links.append( main.ONOScli4.links() )
959 links.append( main.ONOScli5.links() )
960 links.append( main.ONOScli6.links() )
961 links.append( main.ONOScli7.links() )
Jon Hall94fd0472014-12-08 11:52:42 -0800962 clusters = []
963 clusters.append( main.ONOScli1.clusters() )
964 clusters.append( main.ONOScli2.clusters() )
965 clusters.append( main.ONOScli3.clusters() )
966 clusters.append( main.ONOScli4.clusters() )
967 clusters.append( main.ONOScli5.clusters() )
968 clusters.append( main.ONOScli6.clusters() )
969 clusters.append( main.ONOScli7.clusters() )
Jon Hall529a37f2015-01-28 10:02:00 -0800970 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -0800971
Jon Hall6aec96b2015-01-19 14:49:31 -0800972 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -0800973 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800974 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800975 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -0800976 if "Error" not in hosts[ controller ]:
977 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -0800978 continue
Jon Hall6aec96b2015-01-19 14:49:31 -0800979 else: # hosts not consistent
980 main.log.report( "hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -0800981 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -0800982 " is inconsistent with ONOS1" )
983 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800984 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800985
986 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800987 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -0800988 controllerStr )
989 consistentHostsResult = main.FALSE
990 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -0800991 " hosts response: " +
992 repr( hosts[ controller ] ) )
993 utilities.assert_equals(
994 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800995 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800996 onpass="Hosts view is consistent across all ONOS nodes",
997 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -0800998
Jon Hall6aec96b2015-01-19 14:49:31 -0800999 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001000 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001001 for controller in range( len( clusters ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001002 if "Error" not in clusters[ controller ]:
1003 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001004 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001005 else: # clusters not consistent
1006 main.log.report( "clusters from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001007 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001008 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001009 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001010
1011 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001012 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001013 "from ONOS" + controllerStr )
1014 consistentClustersResult = main.FALSE
1015 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001016 " clusters response: " +
1017 repr( clusters[ controller ] ) )
1018 utilities.assert_equals(
1019 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001020 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001021 onpass="Clusters view is consistent across all ONOS nodes",
1022 onfail="ONOS nodes have different views of clusters" )
1023 # there should always only be one cluster
Jon Hall8f89dda2015-01-22 16:03:33 -08001024 numClusters = len( json.loads( clusters[ 0 ] ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001025 utilities.assert_equals(
1026 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001027 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001028 onpass="ONOS shows 1 SCC",
1029 onfail="ONOS shows " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001030 str( numClusters ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001031 " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001032
Jon Hall6aec96b2015-01-19 14:49:31 -08001033 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001034 devicesResults = main.TRUE
1035 portsResults = main.TRUE
1036 linksResults = main.TRUE
1037 for controller in range( numControllers ):
1038 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001039 if devices[ controller ] or "Error" not in devices[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001040 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001041 MNTopo,
1042 json.loads(
1043 devices[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001044 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001045 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001046 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001047 actual=currentDevicesResult,
1048 onpass="ONOS" + controllerStr +
1049 " Switches view is correct",
1050 onfail="ONOS" + controllerStr +
1051 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001052
Jon Hall6aec96b2015-01-19 14:49:31 -08001053 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001054 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001055 MNTopo,
1056 json.loads(
1057 ports[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001058 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001059 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001060 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001061 actual=currentPortsResult,
1062 onpass="ONOS" + controllerStr +
1063 " ports view is correct",
1064 onfail="ONOS" + controllerStr +
1065 " ports view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001066
Jon Hall6aec96b2015-01-19 14:49:31 -08001067 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001068 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001069 MNTopo,
1070 json.loads(
1071 links[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001072 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001073 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001074 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001075 actual=currentLinksResult,
1076 onpass="ONOS" + controllerStr +
1077 " links view is correct",
1078 onfail="ONOS" + controllerStr +
1079 " links view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001080
Jon Hall8f89dda2015-01-22 16:03:33 -08001081 devicesResults = devicesResults and currentDevicesResult
1082 portsResults = portsResults and currentPortsResult
1083 linksResults = linksResults and currentLinksResult
Jon Hall73cf9cc2014-11-20 22:28:38 -08001084
Jon Hall8f89dda2015-01-22 16:03:33 -08001085 topoResult = devicesResults and portsResults and linksResults\
Jon Hall529a37f2015-01-28 10:02:00 -08001086 and consistentHostsResult and consistentClustersResult
Jon Hall8f89dda2015-01-22 16:03:33 -08001087 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1088 onpass="Topology Check Test successful",
1089 onfail="Topology Check Test NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001090
Jon Hall8f89dda2015-01-22 16:03:33 -08001091 finalAssert = main.TRUE
1092 finalAssert = finalAssert and topoResult and flowCheck \
1093 and intentCheck and consistentMastership and rolesNotNull
1094 utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
1095 onpass="State check successful",
1096 onfail="State check NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001097
Jon Hall6aec96b2015-01-19 14:49:31 -08001098 def CASE6( self, main ):
1099 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001100 The Failure case.
Jon Hall6aec96b2015-01-19 14:49:31 -08001101 """
1102 main.log.report( "Restart entire ONOS cluster" )
1103 main.log.case( "Restart entire ONOS cluster" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001104 main.ONOSbench.onosKill( ONOS1Ip )
1105 main.ONOSbench.onosKill( ONOS2Ip )
1106 main.ONOSbench.onosKill( ONOS3Ip )
1107 main.ONOSbench.onosKill( ONOS4Ip )
1108 main.ONOSbench.onosKill( ONOS5Ip )
1109 main.ONOSbench.onosKill( ONOS6Ip )
1110 main.ONOSbench.onosKill( ONOS7Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001111
Jon Hall6aec96b2015-01-19 14:49:31 -08001112 main.step( "Checking if ONOS is up yet" )
Jon Hallffb386d2014-11-21 13:43:38 -08001113 count = 0
Jon Hall8f89dda2015-01-22 16:03:33 -08001114 onosIsupResult = main.FALSE
1115 while onosIsupResult == main.FALSE and count < 10:
1116 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
1117 onos2Isup = main.ONOSbench.isup( ONOS2Ip )
1118 onos3Isup = main.ONOSbench.isup( ONOS3Ip )
1119 onos4Isup = main.ONOSbench.isup( ONOS4Ip )
1120 onos5Isup = main.ONOSbench.isup( ONOS5Ip )
1121 onos6Isup = main.ONOSbench.isup( ONOS6Ip )
1122 onos7Isup = main.ONOSbench.isup( ONOS7Ip )
1123 onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
1124 and onos4Isup and onos5Isup and onos6Isup and onos7Isup
Jon Hallffb386d2014-11-21 13:43:38 -08001125 count = count + 1
Jon Hall73cf9cc2014-11-20 22:28:38 -08001126 # TODO: if it becomes an issue, we can retry this step a few times
1127
Jon Hall8f89dda2015-01-22 16:03:33 -08001128 cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
1129 cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
1130 cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
1131 cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
1132 cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
1133 cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
1134 cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
1135 cliResults = cliResult1 and cliResult2 and cliResult3\
1136 and cliResult4 and cliResult5 and cliResult6\
1137 and cliResult7
Jon Hall73cf9cc2014-11-20 22:28:38 -08001138
Jon Hall8f89dda2015-01-22 16:03:33 -08001139 caseResults = main.TRUE and onosIsupResult and cliResults
1140 utilities.assert_equals( expect=main.TRUE, actual=caseResults,
1141 onpass="ONOS restart successful",
1142 onfail="ONOS restart NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001143
Jon Hall6aec96b2015-01-19 14:49:31 -08001144 def CASE7( self, main ):
1145 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001146 Check state after ONOS failure
Jon Hall6aec96b2015-01-19 14:49:31 -08001147 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001148 import json
Jon Hall6aec96b2015-01-19 14:49:31 -08001149 main.case( "Running ONOS Constant State Tests" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001150
Jon Hall6aec96b2015-01-19 14:49:31 -08001151 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -08001152 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
1153 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
1154 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
1155 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
1156 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
1157 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
1158 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
1159 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
1160 ONOS3MasterNotNull and ONOS4MasterNotNull and\
1161 ONOS5MasterNotNull and ONOS6MasterNotNull and\
1162 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -08001163 utilities.assert_equals(
1164 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001165 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -08001166 onpass="Each device has a master",
1167 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -08001168
Jon Hall6aec96b2015-01-19 14:49:31 -08001169 main.step( "Check if switch roles are consistent across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001170 ONOS1Mastership = main.ONOScli1.roles()
1171 ONOS2Mastership = main.ONOScli2.roles()
1172 ONOS3Mastership = main.ONOScli3.roles()
1173 ONOS4Mastership = main.ONOScli4.roles()
1174 ONOS5Mastership = main.ONOScli5.roles()
1175 ONOS6Mastership = main.ONOScli6.roles()
1176 ONOS7Mastership = main.ONOScli7.roles()
1177 if "Error" in ONOS1Mastership or not ONOS1Mastership\
1178 or "Error" in ONOS2Mastership or not ONOS2Mastership\
1179 or "Error" in ONOS3Mastership or not ONOS3Mastership\
1180 or "Error" in ONOS4Mastership or not ONOS4Mastership\
1181 or "Error" in ONOS5Mastership or not ONOS5Mastership\
1182 or "Error" in ONOS6Mastership or not ONOS6Mastership\
1183 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -08001184 main.log.error( "Error in getting ONOS mastership" )
1185 main.log.warn( "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001186 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001187 main.log.warn( "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001188 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001189 main.log.warn( "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001190 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001191 main.log.warn( "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001192 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001193 main.log.warn( "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001194 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001195 main.log.warn( "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001196 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001197 main.log.warn( "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001198 repr( ONOS7Mastership ) )
1199 consistentMastership = main.FALSE
1200 elif ONOS1Mastership == ONOS2Mastership\
1201 and ONOS1Mastership == ONOS3Mastership\
1202 and ONOS1Mastership == ONOS4Mastership\
1203 and ONOS1Mastership == ONOS5Mastership\
1204 and ONOS1Mastership == ONOS6Mastership\
1205 and ONOS1Mastership == ONOS7Mastership:
1206 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001207 main.log.report(
1208 "Switch roles are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001209 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001210 main.log.warn( "ONOS1 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001211 json.loads( ONOS1Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001212 separators=( ',', ': ' ) ) )
1213 main.log.warn( "ONOS2 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001214 json.loads( ONOS2Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001215 separators=( ',', ': ' ) ) )
1216 main.log.warn( "ONOS3 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001217 json.loads( ONOS3Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001218 separators=( ',', ': ' ) ) )
1219 main.log.warn( "ONOS4 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001220 json.loads( ONOS4Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001221 separators=( ',', ': ' ) ) )
1222 main.log.warn( "ONOS5 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001223 json.loads( ONOS5Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001224 separators=( ',', ': ' ) ) )
1225 main.log.warn( "ONOS6 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001226 json.loads( ONOS6Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001227 separators=( ',', ': ' ) ) )
1228 main.log.warn( "ONOS7 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001229 json.loads( ONOS7Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001230 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001231 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001232 utilities.assert_equals(
1233 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001234 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -08001235 onpass="Switch roles are consistent across all ONOS nodes",
1236 onfail="ONOS nodes have different views of switch roles" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001237
1238 description2 = "Compare switch roles from before failure"
Jon Hall6aec96b2015-01-19 14:49:31 -08001239 main.step( description2 )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001240
Jon Hall8f89dda2015-01-22 16:03:33 -08001241 currentJson = json.loads( ONOS1Mastership )
1242 oldJson = json.loads( mastershipState )
1243 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001244 for i in range( 1, 29 ):
1245 switchDPID = str(
1246 main.Mininet1.getSwitchDPID(
1247 switch="s" +
1248 str( i ) ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001249
Jon Hall8f89dda2015-01-22 16:03:33 -08001250 current = [ switch[ 'master' ] for switch in currentJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001251 if switchDPID in switch[ 'id' ] ]
Jon Hall8f89dda2015-01-22 16:03:33 -08001252 old = [ switch[ 'master' ] for switch in oldJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001253 if switchDPID in switch[ 'id' ] ]
Jon Hall73cf9cc2014-11-20 22:28:38 -08001254 if current == old:
Jon Hall8f89dda2015-01-22 16:03:33 -08001255 mastershipCheck = mastershipCheck and main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001256 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001257 main.log.warn( "Mastership of switch %s changed" % switchDPID )
Jon Hall8f89dda2015-01-22 16:03:33 -08001258 mastershipCheck = main.FALSE
1259 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001260 main.log.report( "Mastership of Switches was not changed" )
1261 utilities.assert_equals(
1262 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001263 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001264 onpass="Mastership of Switches was not changed",
1265 onfail="Mastership of some switches changed" )
1266 # NOTE: we expect mastership to change on controller failure
Jon Hall8f89dda2015-01-22 16:03:33 -08001267 mastershipCheck = mastershipCheck and consistentMastership
Jon Hall73cf9cc2014-11-20 22:28:38 -08001268
Jon Hall6aec96b2015-01-19 14:49:31 -08001269 main.step( "Get the intents and compare across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001270 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1271 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
1272 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
1273 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
1274 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
1275 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
1276 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
1277 intentCheck = main.FALSE
1278 if "Error" in ONOS1Intents or not ONOS1Intents\
1279 or "Error" in ONOS2Intents or not ONOS2Intents\
1280 or "Error" in ONOS3Intents or not ONOS3Intents\
1281 or "Error" in ONOS4Intents or not ONOS4Intents\
1282 or "Error" in ONOS5Intents or not ONOS5Intents\
1283 or "Error" in ONOS6Intents or not ONOS6Intents\
1284 or "Error" in ONOS7Intents or not ONOS7Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -08001285 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001286 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1287 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
1288 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
1289 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
1290 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
1291 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
1292 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
1293 elif ONOS1Intents == ONOS2Intents\
1294 and ONOS1Intents == ONOS3Intents\
1295 and ONOS1Intents == ONOS4Intents\
1296 and ONOS1Intents == ONOS5Intents\
1297 and ONOS1Intents == ONOS6Intents\
1298 and ONOS1Intents == ONOS7Intents:
1299 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001300 main.log.report( "Intents are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001301 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001302 main.log.warn( "ONOS1 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001303 print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001304 indent=4, separators=( ',', ': ' ) )
1305 main.log.warn( "ONOS2 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001306 print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001307 indent=4, separators=( ',', ': ' ) )
1308 main.log.warn( "ONOS3 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001309 print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001310 indent=4, separators=( ',', ': ' ) )
1311 main.log.warn( "ONOS4 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001312 print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001313 indent=4, separators=( ',', ': ' ) )
1314 main.log.warn( "ONOS5 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001315 print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001316 indent=4, separators=( ',', ': ' ) )
1317 main.log.warn( "ONOS6 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001318 print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001319 indent=4, separators=( ',', ': ' ) )
1320 main.log.warn( "ONOS7 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001321 print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001322 indent=4, separators=( ',', ': ' ) )
1323 utilities.assert_equals(
1324 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001325 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001326 onpass="Intents are consistent across all ONOS nodes",
1327 onfail="ONOS nodes have different views of intents" )
Jon Hall1b8f54a2015-02-04 13:24:20 -08001328 # Print the intent states
1329 intents = []
1330 intents.append( ONOS1Intents )
1331 intents.append( ONOS2Intents )
1332 intents.append( ONOS3Intents )
1333 intents.append( ONOS4Intents )
1334 intents.append( ONOS5Intents )
1335 intents.append( ONOS6Intents )
1336 intents.append( ONOS7Intents )
1337 intentStates = []
1338 for node in intents: # Iter through ONOS nodes
1339 nodeStates = []
1340 for intent in json.loads( node ): # Iter through intents of a node
1341 nodeStates.append( intent[ 'state' ] )
1342 intentStates.append( nodeStates )
1343 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1344 main.log.info( dict( out ) )
1345
Jon Hall73cf9cc2014-11-20 22:28:38 -08001346
Jon Hall6aec96b2015-01-19 14:49:31 -08001347 # NOTE: Hazelcast has no durability, so intents are lost across system
1348 # restarts
1349 """
1350 main.step( "Compare current intents with intents before the failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001351 # NOTE: this requires case 5 to pass for intentState to be set.
Jon Hall94fd0472014-12-08 11:52:42 -08001352 # maybe we should stop the test if that fails?
Jon Hall1b8f54a2015-02-04 13:24:20 -08001353 sameIntents = main.TRUE
1354 if intentState and intentState == ONOS1Intents:
Jon Hall8f89dda2015-01-22 16:03:33 -08001355 sameIntents = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001356 main.log.report( "Intents are consistent with before failure" )
1357 # TODO: possibly the states have changed? we may need to figure out
1358 # what the aceptable states are
Jon Hall73cf9cc2014-11-20 22:28:38 -08001359 else:
Jon Hall669173b2014-12-17 11:36:30 -08001360 try:
Jon Hall6aec96b2015-01-19 14:49:31 -08001361 main.log.warn( "ONOS1 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001362 print json.dumps( json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -08001363 sort_keys=True, indent=4,
1364 separators=( ',', ': ' ) )
Jon Hall669173b2014-12-17 11:36:30 -08001365 except:
1366 pass
Jon Hall8f89dda2015-01-22 16:03:33 -08001367 sameIntents = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001368 utilities.assert_equals(
1369 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001370 actual=sameIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001371 onpass="Intents are consistent with before failure",
1372 onfail="The Intents changed during failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001373 intentCheck = intentCheck and sameIntents
Jon Hall6aec96b2015-01-19 14:49:31 -08001374 """
1375 main.step( "Get the OF Table entries and compare to before " +
1376 "component failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001377 FlowTables = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001378 flows2 = []
1379 for i in range( 28 ):
1380 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001381 tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
1382 flows2.append( tmpFlows )
1383 tempResult = main.Mininet2.flowComp(
Jon Hall6aec96b2015-01-19 14:49:31 -08001384 flow1=flows[ i ],
Jon Hall8f89dda2015-01-22 16:03:33 -08001385 flow2=tmpFlows )
1386 FlowTables = FlowTables and tempResult
1387 if FlowTables == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001388 main.log.info( "Differences in flow table for switch: s" +
1389 str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001390 if FlowTables == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001391 main.log.report( "No changes were found in the flow tables" )
1392 utilities.assert_equals(
1393 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001394 actual=FlowTables,
Jon Hall6aec96b2015-01-19 14:49:31 -08001395 onpass="No changes were found in the flow tables",
1396 onfail="Changes were found in the flow tables" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001397
Jon Hall6aec96b2015-01-19 14:49:31 -08001398 main.step( "Check the continuous pings to ensure that no packets " +
1399 "were dropped during component failure" )
1400 # FIXME: This check is always failing. Investigate cause
1401 # NOTE: this may be something to do with file permsissions
Jon Hall73cf9cc2014-11-20 22:28:38 -08001402 # or slight change in format
Jon Hall6aec96b2015-01-19 14:49:31 -08001403 main.Mininet2.pingKill(
1404 main.params[ 'TESTONUSER' ],
1405 main.params[ 'TESTONIP' ] )
Jon Hall8f89dda2015-01-22 16:03:33 -08001406 LossInPings = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001407 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1408 for i in range( 8, 18 ):
1409 main.log.info(
1410 "Checking for a loss in pings along flow from s" +
1411 str( i ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001412 LossInPings = main.Mininet2.checkForLoss(
Jon Hall6aec96b2015-01-19 14:49:31 -08001413 "/tmp/ping.h" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001414 str( i ) ) or LossInPings
1415 if LossInPings == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001416 main.log.info( "Loss in ping detected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001417 elif LossInPings == main.ERROR:
Jon Hall6aec96b2015-01-19 14:49:31 -08001418 main.log.info( "There are multiple mininet process running" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001419 elif LossInPings == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001420 main.log.info( "No Loss in the pings" )
1421 main.log.report( "No loss of dataplane connectivity" )
1422 utilities.assert_equals(
1423 expect=main.FALSE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001424 actual=LossInPings,
Jon Hall6aec96b2015-01-19 14:49:31 -08001425 onpass="No Loss of connectivity",
1426 onfail="Loss of dataplane connectivity detected" )
1427 # NOTE: Since intents are not persisted with Hazelcast, we expect this
Jon Hall8f89dda2015-01-22 16:03:33 -08001428 LossInPings = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001429
Jon Hall6aec96b2015-01-19 14:49:31 -08001430 # Test of LeadershipElection
Jon Hall8f89dda2015-01-22 16:03:33 -08001431 leaderList = []
1432 leaderResult = main.TRUE
1433 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001434 # loop through ONOScli handlers
1435 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001436 leaderN = node.electionTestLeader()
1437 leaderList.append( leaderN )
Jon Hall669173b2014-12-17 11:36:30 -08001438 if leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001439 # error in response
1440 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001441 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001442 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001443 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001444 elif leaderN is None:
1445 main.log.report( "ONOS" + str( controller ) +
1446 " shows no leader for the election-app was" +
1447 " elected after the old one died" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001448 leaderResult = main.FALSE
1449 if len( set( leaderList ) ) != 1:
1450 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001451 main.log.error(
1452 "Inconsistent view of leader for the election test app" )
1453 # TODO: print the list
Jon Hall8f89dda2015-01-22 16:03:33 -08001454 if leaderResult:
1455 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001456 "view of leader across listeners and a new " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001457 "leader was re-elected if applicable )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001458 utilities.assert_equals(
1459 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001460 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001461 onpass="Leadership election passed",
1462 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08001463
Jon Hall8f89dda2015-01-22 16:03:33 -08001464 result = ( mastershipCheck and intentCheck and FlowTables and
1465 ( not LossInPings ) and rolesNotNull and leaderResult )
Jon Hall6aec96b2015-01-19 14:49:31 -08001466 result = int( result )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001467 if result == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001468 main.log.report( "Constant State Tests Passed" )
1469 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall8f89dda2015-01-22 16:03:33 -08001470 onpass="Constant State Tests Passed",
1471 onfail="Constant state tests failed" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001472
Jon Hall6aec96b2015-01-19 14:49:31 -08001473 def CASE8( self, main ):
1474 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001475 Compare topo
Jon Hall6aec96b2015-01-19 14:49:31 -08001476 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001477 import sys
Jon Hall6aec96b2015-01-19 14:49:31 -08001478 # FIXME add this path to params
1479 sys.path.append( "/home/admin/sts" )
1480 # assumes that sts is already in you PYTHONPATH
1481 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -08001482 import json
1483 import time
1484
Jon Hall6aec96b2015-01-19 14:49:31 -08001485 description = "Compare ONOS Topology view to Mininet topology"
1486 main.case( description )
1487 main.log.report( description )
1488 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001489 ctrls = []
1490 count = 1
1491 while True:
1492 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -08001493 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
1494 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
1495 temp = temp + ( "ONOS" + str( count ), )
1496 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
1497 temp = temp + \
1498 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
1499 ctrls.append( temp )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001500 count = count + 1
1501 else:
1502 break
Jon Hall6aec96b2015-01-19 14:49:31 -08001503 MNTopo = TestONTopology(
1504 main.Mininet1,
1505 ctrls ) # can also add Intent API info for intent operations
Jon Hall73cf9cc2014-11-20 22:28:38 -08001506
Jon Hall6aec96b2015-01-19 14:49:31 -08001507 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001508 devicesResults = main.TRUE
1509 portsResults = main.TRUE
1510 linksResults = main.TRUE
1511 topoResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001512 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -08001513 count = 0
Jon Hall6aec96b2015-01-19 14:49:31 -08001514 main.step( "Collecting topology information from ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001515 startTime = time.time()
Jon Hall21270ac2015-02-16 17:59:55 -08001516 # Give time for Gossip to work
Jon Hall8f89dda2015-01-22 16:03:33 -08001517 while topoResult == main.FALSE and elapsed < 60:
Jon Hallffb386d2014-11-21 13:43:38 -08001518 count = count + 1
Jon Hall94fd0472014-12-08 11:52:42 -08001519 if count > 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08001520 # TODO: Depricate STS usage
1521 MNTopo = TestONTopology(
1522 main.Mininet1,
1523 ctrls )
Jon Hall8f89dda2015-01-22 16:03:33 -08001524 cliStart = time.time()
Jon Hall94fd0472014-12-08 11:52:42 -08001525 devices = []
1526 devices.append( main.ONOScli1.devices() )
1527 devices.append( main.ONOScli2.devices() )
1528 devices.append( main.ONOScli3.devices() )
1529 devices.append( main.ONOScli4.devices() )
1530 devices.append( main.ONOScli5.devices() )
1531 devices.append( main.ONOScli6.devices() )
1532 devices.append( main.ONOScli7.devices() )
1533 hosts = []
Jon Hall529a37f2015-01-28 10:02:00 -08001534 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1535 hosts.append( json.loads( main.ONOScli2.hosts() ) )
1536 hosts.append( json.loads( main.ONOScli3.hosts() ) )
1537 hosts.append( json.loads( main.ONOScli4.hosts() ) )
1538 hosts.append( json.loads( main.ONOScli5.hosts() ) )
1539 hosts.append( json.loads( main.ONOScli6.hosts() ) )
1540 hosts.append( json.loads( main.ONOScli7.hosts() ) )
1541 for controller in range( 0, len( hosts ) ):
1542 controllerStr = str( controller + 1 )
1543 for host in hosts[ controller ]:
Jon Hall529a37f2015-01-28 10:02:00 -08001544 if host[ 'ips' ] == []:
1545 main.log.error(
1546 "DEBUG:Error with host ips on controller" +
1547 controllerStr + ": " + str( host ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001548 ports = []
1549 ports.append( main.ONOScli1.ports() )
1550 ports.append( main.ONOScli2.ports() )
1551 ports.append( main.ONOScli3.ports() )
1552 ports.append( main.ONOScli4.ports() )
1553 ports.append( main.ONOScli5.ports() )
1554 ports.append( main.ONOScli6.ports() )
1555 ports.append( main.ONOScli7.ports() )
1556 links = []
1557 links.append( main.ONOScli1.links() )
1558 links.append( main.ONOScli2.links() )
1559 links.append( main.ONOScli3.links() )
1560 links.append( main.ONOScli4.links() )
1561 links.append( main.ONOScli5.links() )
1562 links.append( main.ONOScli6.links() )
1563 links.append( main.ONOScli7.links() )
1564 clusters = []
1565 clusters.append( main.ONOScli1.clusters() )
1566 clusters.append( main.ONOScli2.clusters() )
1567 clusters.append( main.ONOScli3.clusters() )
1568 clusters.append( main.ONOScli4.clusters() )
1569 clusters.append( main.ONOScli5.clusters() )
1570 clusters.append( main.ONOScli6.clusters() )
1571 clusters.append( main.ONOScli7.clusters() )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001572
Jon Hall8f89dda2015-01-22 16:03:33 -08001573 elapsed = time.time() - startTime
1574 cliTime = time.time() - cliStart
1575 print "CLI time: " + str( cliTime )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001576
Jon Hall8f89dda2015-01-22 16:03:33 -08001577 for controller in range( numControllers ):
1578 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001579 if devices[ controller ] or "Error" not in devices[
1580 controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001581 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001582 MNTopo,
1583 json.loads(
1584 devices[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001585 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001586 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001587 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001588 actual=currentDevicesResult,
1589 onpass="ONOS" + controllerStr +
1590 " Switches view is correct",
1591 onfail="ONOS" + controllerStr +
1592 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001593
Jon Hall6aec96b2015-01-19 14:49:31 -08001594 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001595 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001596 MNTopo,
1597 json.loads(
1598 ports[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001599 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001600 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001601 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001602 actual=currentPortsResult,
1603 onpass="ONOS" + controllerStr +
1604 " ports view is correct",
1605 onfail="ONOS" + controllerStr +
1606 " ports view is incorrect" )
Jon Hall94fd0472014-12-08 11:52:42 -08001607
Jon Hall6aec96b2015-01-19 14:49:31 -08001608 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001609 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001610 MNTopo,
1611 json.loads(
1612 links[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001613 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001614 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001615 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001616 actual=currentLinksResult,
1617 onpass="ONOS" + controllerStr +
1618 " links view is correct",
1619 onfail="ONOS" + controllerStr +
1620 " links view is incorrect" )
1621 devicesResults = devicesResults and currentDevicesResult
1622 portsResults = portsResults and currentPortsResult
1623 linksResults = linksResults and currentLinksResult
Jon Hall94fd0472014-12-08 11:52:42 -08001624
Jon Hall529a37f2015-01-28 10:02:00 -08001625 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001626
Jon Hall6aec96b2015-01-19 14:49:31 -08001627 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -08001628 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001629 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001630 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001631 if "Error" not in hosts[ controller ]:
1632 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001633 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001634 else: # hosts not consistent
Jon Hall8f89dda2015-01-22 16:03:33 -08001635 main.log.report( "hosts from ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001636 " is inconsistent with ONOS1" )
1637 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001638 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001639
1640 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001641 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001642 controllerStr )
1643 consistentHostsResult = main.FALSE
1644 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001645 " hosts response: " +
1646 repr( hosts[ controller ] ) )
1647 utilities.assert_equals(
1648 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001649 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001650 onpass="Hosts view is consistent across all ONOS nodes",
1651 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001652
Jon Hall6aec96b2015-01-19 14:49:31 -08001653 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001654 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001655 for controller in range( len( clusters ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001656 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001657 if "Error" not in clusters[ controller ]:
1658 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001659 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001660 else: # clusters not consistent
1661 main.log.report( "clusters from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001662 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001663 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001664 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001665
1666 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001667 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001668 "from ONOS" + controllerStr )
1669 consistentClustersResult = main.FALSE
1670 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001671 " clusters response: " +
1672 repr( clusters[ controller ] ) )
1673 utilities.assert_equals(
1674 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001675 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001676 onpass="Clusters view is consistent across all ONOS nodes",
1677 onfail="ONOS nodes have different views of clusters" )
1678 # there should always only be one cluster
Jon Hall8f89dda2015-01-22 16:03:33 -08001679 numClusters = len( json.loads( clusters[ 0 ] ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001680 utilities.assert_equals(
1681 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001682 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001683 onpass="ONOS shows 1 SCC",
1684 onfail="ONOS shows " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001685 str( numClusters ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001686 " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001687
Jon Hall8f89dda2015-01-22 16:03:33 -08001688 topoResult = ( devicesResults and portsResults and linksResults
1689 and consistentHostsResult
Jon Hall529a37f2015-01-28 10:02:00 -08001690 and consistentClustersResult )
Jon Hall94fd0472014-12-08 11:52:42 -08001691
Jon Hall8f89dda2015-01-22 16:03:33 -08001692 topoResult = topoResult and int( count <= 2 )
1693 note = "note it takes about " + str( int( cliTime ) ) + \
1694 " seconds for the test to make all the cli calls to fetch " +\
1695 "the topology from each ONOS instance"
Jon Hall1b8f54a2015-02-04 13:24:20 -08001696 main.log.info(
Jon Hall8f89dda2015-01-22 16:03:33 -08001697 "Very crass estimate for topology discovery/convergence( " +
1698 str( note ) + " ): " + str( elapsed ) + " seconds, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001699 str( count ) + " tries" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001700 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1701 onpass="Topology Check Test successful",
1702 onfail="Topology Check Test NOT successful" )
1703 if topoResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001704 main.log.report( "ONOS topology view matches Mininet topology" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001705
Jon Hall6aec96b2015-01-19 14:49:31 -08001706 def CASE9( self, main ):
1707 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001708 Link s3-s28 down
Jon Hall6aec96b2015-01-19 14:49:31 -08001709 """
1710 import time
1711 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001712
Jon Hall8f89dda2015-01-22 16:03:33 -08001713 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001714
Jon Hall6aec96b2015-01-19 14:49:31 -08001715 description = "Turn off a link to ensure that Link Discovery " +\
Jon Hall8f89dda2015-01-22 16:03:33 -08001716 "is working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001717 main.log.report( description )
1718 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001719
Jon Hall6aec96b2015-01-19 14:49:31 -08001720 main.step( "Kill Link between s3 and s28" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001721 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001722 main.log.info(
1723 "Waiting " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001724 str( linkSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001725 " seconds for link down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001726 time.sleep( linkSleep )
1727 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
1728 onpass="Link down succesful",
1729 onfail="Failed to bring link down" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001730 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -08001731
Jon Hall6aec96b2015-01-19 14:49:31 -08001732 def CASE10( self, main ):
1733 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001734 Link s3-s28 up
Jon Hall6aec96b2015-01-19 14:49:31 -08001735 """
1736 import time
1737 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001738
Jon Hall8f89dda2015-01-22 16:03:33 -08001739 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001740
Jon Hall6aec96b2015-01-19 14:49:31 -08001741 description = "Restore a link to ensure that Link Discovery is " + \
Jon Hall8f89dda2015-01-22 16:03:33 -08001742 "working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001743 main.log.report( description )
1744 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001745
Jon Hall6aec96b2015-01-19 14:49:31 -08001746 main.step( "Bring link between s3 and s28 back up" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001747 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001748 main.log.info(
1749 "Waiting " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001750 str( linkSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001751 " seconds for link up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001752 time.sleep( linkSleep )
1753 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
1754 onpass="Link up succesful",
1755 onfail="Failed to bring link up" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001756 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -08001757
Jon Hall6aec96b2015-01-19 14:49:31 -08001758 def CASE11( self, main ):
1759 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001760 Switch Down
Jon Hall6aec96b2015-01-19 14:49:31 -08001761 """
1762 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001763 import time
1764
Jon Hall8f89dda2015-01-22 16:03:33 -08001765 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001766
1767 description = "Killing a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001768 main.log.report( description )
1769 main.case( description )
1770 switch = main.params[ 'kill' ][ 'switch' ]
1771 switchDPID = main.params[ 'kill' ][ 'dpid' ]
Jon Hall73cf9cc2014-11-20 22:28:38 -08001772
Jon Hall6aec96b2015-01-19 14:49:31 -08001773 # TODO: Make this switch parameterizable
1774 main.step( "Kill " + switch )
1775 main.log.report( "Deleting " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08001776 main.Mininet1.delSwitch( switch )
1777 main.log.info( "Waiting " + str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001778 " seconds for switch down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001779 time.sleep( switchSleep )
1780 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001781 # Peek at the deleted switch
1782 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001783 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001784 if device and device[ 'available' ] is False:
Jon Hall94fd0472014-12-08 11:52:42 -08001785 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001786 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall8f89dda2015-01-22 16:03:33 -08001787 onpass="Kill switch succesful",
1788 onfail="Failed to kill switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001789
Jon Hall6aec96b2015-01-19 14:49:31 -08001790 def CASE12( self, main ):
1791 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001792 Switch Up
Jon Hall6aec96b2015-01-19 14:49:31 -08001793 """
1794 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08001795 import time
Jon Hall669173b2014-12-17 11:36:30 -08001796
Jon Hall8f89dda2015-01-22 16:03:33 -08001797 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall6aec96b2015-01-19 14:49:31 -08001798 switch = main.params[ 'kill' ][ 'switch' ]
1799 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1800 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hall73cf9cc2014-11-20 22:28:38 -08001801 description = "Adding a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001802 main.log.report( description )
1803 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001804
Jon Hall6aec96b2015-01-19 14:49:31 -08001805 main.step( "Add back " + switch )
1806 main.log.report( "Adding back " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08001807 main.Mininet1.addSwitch( switch, dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001808 # TODO: New dpid or same? Ask Thomas?
1809 for peer in links:
Jon Hall8f89dda2015-01-22 16:03:33 -08001810 main.Mininet1.addLink( switch, peer )
1811 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -08001812 sw=switch.split( 's' )[ 1 ],
Jon Hall8f89dda2015-01-22 16:03:33 -08001813 count=numControllers,
1814 ip1=ONOS1Ip,
1815 port1=ONOS1Port,
1816 ip2=ONOS2Ip,
1817 port2=ONOS2Port,
1818 ip3=ONOS3Ip,
1819 port3=ONOS3Port,
1820 ip4=ONOS4Ip,
1821 port4=ONOS4Port,
1822 ip5=ONOS5Ip,
1823 port5=ONOS5Port,
1824 ip6=ONOS6Ip,
1825 port6=ONOS6Port,
1826 ip7=ONOS7Ip,
1827 port7=ONOS7Port )
Jon Hall6aec96b2015-01-19 14:49:31 -08001828 main.log.info(
1829 "Waiting " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001830 str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001831 " seconds for switch up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001832 time.sleep( switchSleep )
1833 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001834 # Peek at the deleted switch
1835 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001836 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001837 if device and device[ 'available' ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001838 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001839 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall8f89dda2015-01-22 16:03:33 -08001840 onpass="add switch succesful",
1841 onfail="Failed to add switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001842
Jon Hall6aec96b2015-01-19 14:49:31 -08001843 def CASE13( self, main ):
1844 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001845 Clean up
Jon Hall6aec96b2015-01-19 14:49:31 -08001846 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001847 import os
1848 import time
Jon Hall6aec96b2015-01-19 14:49:31 -08001849 # TODO: make use of this elsewhere
1850 ips = []
Jon Hall8f89dda2015-01-22 16:03:33 -08001851 ips.append( ONOS1Ip )
1852 ips.append( ONOS2Ip )
1853 ips.append( ONOS3Ip )
1854 ips.append( ONOS4Ip )
1855 ips.append( ONOS5Ip )
1856 ips.append( ONOS6Ip )
1857 ips.append( ONOS7Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -08001858
1859 # printing colors to terminal
Jon Hall669173b2014-12-17 11:36:30 -08001860 colors = {}
Jon Hall6aec96b2015-01-19 14:49:31 -08001861 colors[ 'cyan' ] = '\033[96m'
1862 colors[ 'purple' ] = '\033[95m'
1863 colors[ 'blue' ] = '\033[94m'
1864 colors[ 'green' ] = '\033[92m'
1865 colors[ 'yellow' ] = '\033[93m'
1866 colors[ 'red' ] = '\033[91m'
1867 colors[ 'end' ] = '\033[0m'
Jon Hall73cf9cc2014-11-20 22:28:38 -08001868 description = "Test Cleanup"
Jon Hall6aec96b2015-01-19 14:49:31 -08001869 main.log.report( description )
1870 main.case( description )
1871 main.step( "Killing tcpdumps" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001872 main.Mininet2.stopTcpdump()
Jon Hall73cf9cc2014-11-20 22:28:38 -08001873
Jon Hall6aec96b2015-01-19 14:49:31 -08001874 main.step( "Checking ONOS Logs for errors" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001875 for i in range( 7 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001876 print colors[ 'purple' ] + "Checking logs for errors on " + \
Jon Hall8f89dda2015-01-22 16:03:33 -08001877 "ONOS" + str( i + 1 ) + ":" + colors[ 'end' ]
1878 print main.ONOSbench.checkLogs( ips[ i ] )
Jon Hall94fd0472014-12-08 11:52:42 -08001879
Jon Hall6aec96b2015-01-19 14:49:31 -08001880 main.step( "Copying MN pcap and ONOS log files to test station" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001881 testname = main.TEST
Jon Hall8f89dda2015-01-22 16:03:33 -08001882 teststationUser = main.params[ 'TESTONUSER' ]
1883 teststationIP = main.params[ 'TESTONIP' ]
Jon Hall6aec96b2015-01-19 14:49:31 -08001884 # NOTE: MN Pcap file is being saved to ~/packet_captures
Jon Hall73cf9cc2014-11-20 22:28:38 -08001885 # scp this file as MN and TestON aren't necessarily the same vm
Jon Hall6aec96b2015-01-19 14:49:31 -08001886 # FIXME: scp
1887 # mn files
1888 # TODO: Load these from params
1889 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08001890 logFolder = "/opt/onos/log/"
1891 logFiles = [ "karaf.log", "karaf.log.1" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08001892 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08001893 dstDir = "~/packet_captures/"
1894 for f in logFiles:
1895 for i in range( 7 ):
1896 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
1897 logFolder + f + " " +
1898 teststationUser + "@" +
1899 teststationIP + ":" +
1900 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001901 "-ONOS" + str( i + 1 ) + "-" +
1902 f )
1903 # std*.log's
1904 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08001905 logFolder = "/opt/onos/var/"
1906 logFiles = [ "stderr.log", "stdout.log" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08001907 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08001908 dstDir = "~/packet_captures/"
1909 for f in logFiles:
1910 for i in range( 7 ):
1911 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
1912 logFolder + f + " " +
1913 teststationUser + "@" +
1914 teststationIP + ":" +
1915 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001916 "-ONOS" + str( i + 1 ) + "-" +
1917 f )
1918 # sleep so scp can finish
1919 time.sleep( 10 )
1920 main.step( "Packing and rotating pcap archives" )
1921 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001922
Jon Hall6aec96b2015-01-19 14:49:31 -08001923 # TODO: actually check something here
1924 utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001925 onpass="Test cleanup successful",
1926 onfail="Test cleanup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001927
Jon Hall6aec96b2015-01-19 14:49:31 -08001928 def CASE14( self, main ):
1929 """
Jon Hall669173b2014-12-17 11:36:30 -08001930 start election app on all onos nodes
Jon Hall6aec96b2015-01-19 14:49:31 -08001931 """
Jon Hall8f89dda2015-01-22 16:03:33 -08001932 leaderResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001933 # install app on onos 1
1934 main.log.info( "Install leadership election app" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001935 main.ONOScli1.featureInstall( "onos-app-election" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001936 # wait for election
1937 # check for leader
Jon Hall8f89dda2015-01-22 16:03:33 -08001938 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001939 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08001940 if leader == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08001941 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08001942 pass
Jon Hall6aec96b2015-01-19 14:49:31 -08001943 elif leader is None:
1944 # No leader elected
1945 main.log.report( "No leader was elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001946 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001947 elif leader == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001948 # error in response
1949 # TODO: add check for "Command not found:" in the driver, this
1950 # means the app isn't loaded
Jon Hall8f89dda2015-01-22 16:03:33 -08001951 main.log.report( "Something is wrong with electionTestLeader" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001952 " function, check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001953 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001954 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001955 # error in response
1956 main.log.report(
Jon Hall8f89dda2015-01-22 16:03:33 -08001957 "Unexpected response from electionTestLeader function:'" +
1958 str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001959 "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001960 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001961
Jon Hall6aec96b2015-01-19 14:49:31 -08001962 # install on other nodes and check for leader.
1963 # Should be onos1 and each app should show the same leader
Jon Hall8f89dda2015-01-22 16:03:33 -08001964 for controller in range( 2, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001965 # loop through ONOScli handlers
1966 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001967 node.featureInstall( "onos-app-election" )
1968 leaderN = node.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001969 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08001970 if leaderN == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08001971 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08001972 pass
1973 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001974 # error in response
1975 # TODO: add check for "Command not found:" in the driver, this
1976 # means the app isn't loaded
1977 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001978 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001979 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001980 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001981 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08001982 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001983 main.log.report( "ONOS" + str( controller ) + " sees " +
1984 str( leaderN ) +
1985 " as the leader of the election app. Leader" +
1986 " should be " +
1987 str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001988 if leaderResult:
Jon Hall6aec96b2015-01-19 14:49:31 -08001989 main.log.report( "Leadership election tests passed( consistent " +
1990 "view of leader across listeners and a leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001991 "was elected )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001992 utilities.assert_equals(
1993 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001994 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001995 onpass="Leadership election passed",
1996 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08001997
Jon Hall6aec96b2015-01-19 14:49:31 -08001998 def CASE15( self, main ):
1999 """
Jon Hall669173b2014-12-17 11:36:30 -08002000 Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -08002001 """
Jon Hall8f89dda2015-01-22 16:03:33 -08002002 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002003 description = "Check that Leadership Election is still functional"
Jon Hall6aec96b2015-01-19 14:49:31 -08002004 main.log.report( description )
2005 main.case( description )
2006 main.step( "Find current leader and withdraw" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002007 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002008 # TODO: do some sanity checking on leader before using it
Jon Hall8f89dda2015-01-22 16:03:33 -08002009 withdrawResult = main.FALSE
2010 if leader == ONOS1Ip:
2011 oldLeader = getattr( main, "ONOScli1" )
2012 elif leader == ONOS2Ip:
2013 oldLeader = getattr( main, "ONOScli2" )
2014 elif leader == ONOS3Ip:
2015 oldLeader = getattr( main, "ONOScli3" )
2016 elif leader == ONOS4Ip:
2017 oldLeader = getattr( main, "ONOScli4" )
2018 elif leader == ONOS5Ip:
2019 oldLeader = getattr( main, "ONOScli5" )
2020 elif leader == ONOS6Ip:
2021 oldLeader = getattr( main, "ONOScli6" )
2022 elif leader == ONOS7Ip:
2023 oldLeader = getattr( main, "ONOScli7" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002024 elif leader is None or leader == main.FALSE:
2025 main.log.report(
2026 "Leader for the election app should be an ONOS node," +
2027 "instead got '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08002028 str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002029 "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002030 leaderResult = main.FALSE
2031 withdrawResult = oldLeader.electionTestWithdraw()
Jon Hall6aec96b2015-01-19 14:49:31 -08002032 utilities.assert_equals(
2033 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002034 actual=withdrawResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002035 onpass="App was withdrawn from election",
2036 onfail="App was not withdrawn from election" )
Jon Hall669173b2014-12-17 11:36:30 -08002037
Jon Hall6aec96b2015-01-19 14:49:31 -08002038 main.step( "Make sure new leader is elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002039 leaderList = []
2040 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002041 # loop through ONOScli handlers
2042 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002043 leaderList.append( node.electionTestLeader() )
2044 for leaderN in leaderList:
Jon Hall669173b2014-12-17 11:36:30 -08002045 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08002046 main.log.report(
2047 "ONOS" +
2048 str( controller ) +
2049 " still sees " +
2050 str( leader ) +
2051 " as leader after they withdrew" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002052 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002053 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002054 # error in response
2055 # TODO: add check for "Command not found:" in the driver, this
2056 # means the app isn't loaded
2057 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002058 "electionTestLeader function, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002059 "check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002060 leaderResult = main.FALSE
2061 consistentLeader = main.FALSE
2062 if len( set( leaderList ) ) == 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08002063 main.log.info( "Each Election-app sees '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08002064 str( leaderList[ 0 ] ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002065 "' as the leader" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002066 consistentLeader = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002067 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002068 main.log.report(
2069 "Inconsistent responses for leader of Election-app:" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002070 for n in range( len( leaderList ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002071 main.log.report( "ONOS" + str( n + 1 ) + " response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002072 str( leaderList[ n ] ) )
2073 if leaderResult:
2074 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002075 "view of leader across listeners and a new " +
2076 "leader was elected when the old leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002077 "resigned )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002078 utilities.assert_equals(
2079 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002080 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002081 onpass="Leadership election passed",
2082 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002083
Jon Hall6aec96b2015-01-19 14:49:31 -08002084 main.step(
Jon Hall8f89dda2015-01-22 16:03:33 -08002085 "Run for election on old leader( just so everyone is in the hat )" )
2086 runResult = oldLeader.electionTestRun()
Jon Hall6aec96b2015-01-19 14:49:31 -08002087 utilities.assert_equals(
2088 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002089 actual=runResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002090 onpass="App re-ran for election",
2091 onfail="App failed to run for election" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002092 if consistentLeader == main.TRUE:
2093 afterRun = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002094 # verify leader didn't just change
Jon Hall8f89dda2015-01-22 16:03:33 -08002095 if afterRun == leaderList[ 0 ]:
2096 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002097 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08002098 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002099 # TODO: assert on run and withdraw results?
Jon Hall669173b2014-12-17 11:36:30 -08002100
Jon Hall6aec96b2015-01-19 14:49:31 -08002101 utilities.assert_equals(
2102 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002103 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002104 onpass="Leadership election passed",
2105 onfail="Something went wrong with Leadership election after " +
2106 "the old leader re-ran for election" )