blob: 0bd05ad5cf69d9aa83f1d3b7e7cfda95bf3d96f6 [file] [log] [blame]
Jon Hall6aec96b2015-01-19 14:49:31 -08001"""
Jon Hall73cf9cc2014-11-20 22:28:38 -08002Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign mastership to controllers
8CASE3: Assign intents
9CASE4: Ping across added host intents
10CASE5: Reading state of ONOS
11CASE6: The Failure case.
12CASE7: Check state after control plane failure
13CASE8: Compare topo
14CASE9: Link s3-s28 down
15CASE10: Link s3-s28 up
16CASE11: Switch down
17CASE12: Switch up
18CASE13: Clean up
Jon Hall669173b2014-12-17 11:36:30 -080019CASE14: start election app on all onos nodes
20CASE15: Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -080021"""
Jon Hall8f89dda2015-01-22 16:03:33 -080022
23
Jon Hall73cf9cc2014-11-20 22:28:38 -080024class HATestClusterRestart:
25
Jon Hall6aec96b2015-01-19 14:49:31 -080026 def __init__( self ):
Jon Hall73cf9cc2014-11-20 22:28:38 -080027 self.default = ''
28
Jon Hall6aec96b2015-01-19 14:49:31 -080029 def CASE1( self, main ):
30 """
Jon Hall73cf9cc2014-11-20 22:28:38 -080031 CASE1 is to compile ONOS and push it to the test machines
32
33 Startup sequence:
Jon Hall73cf9cc2014-11-20 22:28:38 -080034 cell <name>
35 onos-verify-cell
36 NOTE: temporary - onos-remove-raft-logs
Jon Hall58c76b72015-02-23 11:09:24 -080037 onos-uninstall
38 start mininet
39 git pull
40 mvn clean install
41 onos-package
Jon Hall73cf9cc2014-11-20 22:28:38 -080042 onos-install -f
43 onos-wait-for-start
Jon Hall58c76b72015-02-23 11:09:24 -080044 start cli sessions
45 start tcpdump
Jon Hall6aec96b2015-01-19 14:49:31 -080046 """
47 main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
48 "initialization" )
49 main.case( "Setting up test environment" )
50 # TODO: save all the timers and output them for plotting
Jon Hall73cf9cc2014-11-20 22:28:38 -080051
Jon Hall5cfd23c2015-03-19 11:40:57 -070052 # load some variables from the params file
Jon Hall8f89dda2015-01-22 16:03:33 -080053 PULLCODE = False
Jon Hall6aec96b2015-01-19 14:49:31 -080054 if main.params[ 'Git' ] == 'True':
Jon Hall8f89dda2015-01-22 16:03:33 -080055 PULLCODE = True
Jon Hall529a37f2015-01-28 10:02:00 -080056 gitBranch = main.params[ 'branch' ]
Jon Hall8f89dda2015-01-22 16:03:33 -080057 cellName = main.params[ 'ENV' ][ 'cellName' ]
Jon Hall6aec96b2015-01-19 14:49:31 -080058
59 # set global variables
Jon Hall8f89dda2015-01-22 16:03:33 -080060 global ONOS1Port
Jon Hall8f89dda2015-01-22 16:03:33 -080061 global ONOS2Port
Jon Hall8f89dda2015-01-22 16:03:33 -080062 global ONOS3Port
Jon Hall8f89dda2015-01-22 16:03:33 -080063 global ONOS4Port
Jon Hall8f89dda2015-01-22 16:03:33 -080064 global ONOS5Port
Jon Hall8f89dda2015-01-22 16:03:33 -080065 global ONOS6Port
Jon Hall8f89dda2015-01-22 16:03:33 -080066 global ONOS7Port
67 global numControllers
Jon Hall8f89dda2015-01-22 16:03:33 -080068 numControllers = int( main.params[ 'num_controllers' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -080069
Jon Hall5cfd23c2015-03-19 11:40:57 -070070 # FIXME: just get controller port from params?
71 # TODO: do we really need all these?
72 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
73 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
74 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
75 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
76 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
77 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
78 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
79
80 global CLIs
81 CLIs = []
82 global nodes
83 nodes = []
84 for i in range( 1, numControllers + 1 ):
85 CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
86 nodes.append( getattr( main, 'ONOS' + str( i ) ) )
87
Jon Hall6aec96b2015-01-19 14:49:31 -080088 main.step( "Applying cell variable to environment" )
Jon Hall8f89dda2015-01-22 16:03:33 -080089 cellResult = main.ONOSbench.setCell( cellName )
90 verifyResult = main.ONOSbench.verifyCell()
Jon Hall73cf9cc2014-11-20 22:28:38 -080091
Jon Hall6aec96b2015-01-19 14:49:31 -080092 # FIXME:this is short term fix
93 main.log.report( "Removing raft logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -080094 main.ONOSbench.onosRemoveRaftLogs()
Jon Hall5cfd23c2015-03-19 11:40:57 -070095
Jon Hall6aec96b2015-01-19 14:49:31 -080096 main.log.report( "Uninstalling ONOS" )
Jon Hall5cfd23c2015-03-19 11:40:57 -070097 for node in nodes:
98 main.ONOSbench.onosUninstall( node.ip_address )
Jon Hall73cf9cc2014-11-20 22:28:38 -080099
Jon Hall8f89dda2015-01-22 16:03:33 -0800100 cleanInstallResult = main.TRUE
101 gitPullResult = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800102
Jon Hall97f31752015-02-04 12:01:04 -0800103 main.step( "Starting Mininet" )
104 main.Mininet1.startNet( )
105
Jon Hall6aec96b2015-01-19 14:49:31 -0800106 main.step( "Compiling the latest version of ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800107 if PULLCODE:
Jon Hall58c76b72015-02-23 11:09:24 -0800108 main.step( "Git checkout and pull " + gitBranch )
Jon Hall529a37f2015-01-28 10:02:00 -0800109 main.ONOSbench.gitCheckout( gitBranch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800110 gitPullResult = main.ONOSbench.gitPull()
Jon Hall5cfd23c2015-03-19 11:40:57 -0700111 if gitPullResult == main.ERROR:
112 main.log.error( "Error pulling git branch" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800113
Jon Hall6aec96b2015-01-19 14:49:31 -0800114 main.step( "Using mvn clean & install" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800115 cleanInstallResult = main.ONOSbench.cleanInstall()
Jon Hall6aec96b2015-01-19 14:49:31 -0800116 else:
117 main.log.warn( "Did not pull new code so skipping mvn " +
118 "clean install" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800119 main.ONOSbench.getVersion( report=True )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800120
Jon Hall6aec96b2015-01-19 14:49:31 -0800121 main.step( "Creating ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800122 packageResult = main.ONOSbench.onosPackage()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800123
Jon Hall6aec96b2015-01-19 14:49:31 -0800124 main.step( "Installing ONOS package" )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700125 onosInstallResult = main.TRUE
126 for node in nodes:
127 tmpResult = main.ONOSbench.onosInstall( options="-f",
128 node=node.ip_address )
129 onosInstallResult = onosInstallResult and tmpResult
Jon Hall73cf9cc2014-11-20 22:28:38 -0800130
Jon Hall6aec96b2015-01-19 14:49:31 -0800131 main.step( "Checking if ONOS is up yet" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800132 for i in range( 2 ):
Jon Hall5cfd23c2015-03-19 11:40:57 -0700133 onosIsupResult = main.TRUE
134 for node in nodes:
135 started = main.ONOSbench.isup( node.ip_address )
136 if not started:
137 main.log.report( node.name + " didn't start!" )
138 main.ONOSbench.onosStop( node.ip_address )
139 main.ONOSbench.onosStart( node.ip_address )
140 onosIsupResult = onosIsupResult and started
Jon Hall8f89dda2015-01-22 16:03:33 -0800141 if onosIsupResult == main.TRUE:
Jon Hall94fd0472014-12-08 11:52:42 -0800142 break
Jon Hall73cf9cc2014-11-20 22:28:38 -0800143
Jon Hall5cfd23c2015-03-19 11:40:57 -0700144 main.log.step( "Starting ONOS CLI sessions" )
145 cliResults = main.TRUE
146 threads = []
147 for i in range( numControllers ):
148 t = main.Thread( target=CLIs[i].startOnosCli,
149 name="startOnosCli-" + str( i ),
150 args=[nodes[i].ip_address] )
151 threads.append( t )
152 t.start()
153
154 for t in threads:
155 t.join()
156 cliResults = cliResults and t.result
Jon Hall73cf9cc2014-11-20 22:28:38 -0800157
Jon Hall6aec96b2015-01-19 14:49:31 -0800158 main.step( "Start Packet Capture MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800159 main.Mininet2.startTcpdump(
Jon Hall6aec96b2015-01-19 14:49:31 -0800160 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
161 + "-MN.pcap",
162 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
163 port=main.params[ 'MNtcpdump' ][ 'port' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800164
Jon Hall8f89dda2015-01-22 16:03:33 -0800165 case1Result = ( cleanInstallResult and packageResult and
166 cellResult and verifyResult and onosInstallResult
167 and onosIsupResult and cliResults )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800168
Jon Hall8f89dda2015-01-22 16:03:33 -0800169 utilities.assert_equals( expect=main.TRUE, actual=case1Result,
Jon Hall58c76b72015-02-23 11:09:24 -0800170 onpass="Test startup successful",
171 onfail="Test startup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800172
Jon Hall8f89dda2015-01-22 16:03:33 -0800173 if case1Result == main.FALSE:
Jon Hall94fd0472014-12-08 11:52:42 -0800174 main.cleanup()
175 main.exit()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800176
Jon Hall6aec96b2015-01-19 14:49:31 -0800177 def CASE2( self, main ):
178 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800179 Assign mastership to controllers
Jon Hall6aec96b2015-01-19 14:49:31 -0800180 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800181 import re
Jon Hall5cfd23c2015-03-19 11:40:57 -0700182 assert numControllers, "numControllers not defined"
183 assert main, "main not defined"
184 assert utilities.assert_equals, "utilities.assert_equals not defined"
185 assert CLIs, "CLIs not defined"
186 assert nodes, "nodes not defined"
187 assert ONOS1Port, "ONOS1Port not defined"
188 assert ONOS2Port, "ONOS2Port not defined"
189 assert ONOS3Port, "ONOS3Port not defined"
190 assert ONOS4Port, "ONOS4Port not defined"
191 assert ONOS5Port, "ONOS5Port not defined"
192 assert ONOS6Port, "ONOS6Port not defined"
193 assert ONOS7Port, "ONOS7Port not defined"
Jon Hall73cf9cc2014-11-20 22:28:38 -0800194
Jon Hall6aec96b2015-01-19 14:49:31 -0800195 main.log.report( "Assigning switches to controllers" )
196 main.case( "Assigning Controllers" )
197 main.step( "Assign switches to controllers" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800198
Jon Hall5cfd23c2015-03-19 11:40:57 -0700199 # TODO: rewrite this function to take lists of ips and ports?
200 # or list of tuples?
Jon Hall6aec96b2015-01-19 14:49:31 -0800201 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800202 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -0800203 sw=str( i ),
Jon Hall8f89dda2015-01-22 16:03:33 -0800204 count=numControllers,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700205 ip1=nodes[ 0 ].ip_address, port1=ONOS1Port,
206 ip2=nodes[ 1 ].ip_address, port2=ONOS2Port,
207 ip3=nodes[ 2 ].ip_address, port3=ONOS3Port,
208 ip4=nodes[ 3 ].ip_address, port4=ONOS4Port,
209 ip5=nodes[ 4 ].ip_address, port5=ONOS5Port,
210 ip6=nodes[ 5 ].ip_address, port6=ONOS6Port,
211 ip7=nodes[ 6 ].ip_address, port7=ONOS7Port )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800212
Jon Hall8f89dda2015-01-22 16:03:33 -0800213 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800214 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800215 response = main.Mininet1.getSwController( "s" + str( i ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800216 try:
Jon Hall6aec96b2015-01-19 14:49:31 -0800217 main.log.info( str( response ) )
Jon Hallfebb1c72015-03-05 13:30:09 -0800218 except Exception:
Jon Hall6aec96b2015-01-19 14:49:31 -0800219 main.log.info( repr( response ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700220 for node in nodes:
221 if re.search( "tcp:" + node.ip_address, response ):
222 mastershipCheck = mastershipCheck and main.TRUE
223 else:
224 mastershipCheck = main.FALSE
Jon Hall8f89dda2015-01-22 16:03:33 -0800225 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800226 main.log.report( "Switch mastership assigned correctly" )
227 utilities.assert_equals(
228 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800229 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800230 onpass="Switch mastership assigned correctly",
231 onfail="Switches not assigned correctly to controllers" )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700232 #FIXME: turning off because of ONOS-1286
Jon Hall6aec96b2015-01-19 14:49:31 -0800233 # Manually assign mastership to the controller we want
Jon Hall8f89dda2015-01-22 16:03:33 -0800234 roleCall = main.TRUE
235 roleCheck = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -0800236 try:
237 # Assign switch
Jon Hall5cfd23c2015-03-19 11:40:57 -0700238 ip = nodes[ 0 ].ip_address # ONOS1
Jon Hall58c76b72015-02-23 11:09:24 -0800239 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
240 assert deviceId, "No device id for s1 in ONOS"
Jon Hall8f89dda2015-01-22 16:03:33 -0800241 roleCall = roleCall and main.ONOScli1.deviceRole(
242 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700243 ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800244 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700245 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800246 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800247 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800248 roleCheck = roleCheck and main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800249
Jon Hall58c76b72015-02-23 11:09:24 -0800250 # Assign switch
251 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
252 assert deviceId, "No device id for s28 in ONOS"
Jon Hall8f89dda2015-01-22 16:03:33 -0800253 roleCall = roleCall and main.ONOScli1.deviceRole(
254 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700255 ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800256 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700257 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800258 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800259 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800260 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800261
Jon Hall5cfd23c2015-03-19 11:40:57 -0700262 ip = nodes[ 1 ].ip_address # ONOS2
Jon Hall58c76b72015-02-23 11:09:24 -0800263 # Assign switch
264 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
265 assert deviceId, "No device id for s2 in ONOS"
266 roleCall = roleCall and main.ONOScli1.deviceRole(
267 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700268 ip )
Jon Hall58c76b72015-02-23 11:09:24 -0800269 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700270 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall58c76b72015-02-23 11:09:24 -0800271 roleCheck = roleCheck and main.TRUE
272 else:
273 roleCheck = roleCheck and main.FALSE
274
275 # Assign switch
276 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
277 assert deviceId, "No device id for s3 in ONOS"
278 roleCall = roleCall and main.ONOScli1.deviceRole(
279 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700280 ip )
Jon Hall58c76b72015-02-23 11:09:24 -0800281 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700282 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall58c76b72015-02-23 11:09:24 -0800283 roleCheck = roleCheck and main.TRUE
284 else:
285 roleCheck = roleCheck and main.FALSE
286
Jon Hall5cfd23c2015-03-19 11:40:57 -0700287 ip = nodes[ 2 ].ip_address # ONOS3
Jon Hall58c76b72015-02-23 11:09:24 -0800288 # Assign switch
289 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
290 assert deviceId, "No device id for s5 in ONOS"
291 roleCall = roleCall and main.ONOScli1.deviceRole(
292 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700293 ip )
Jon Hall58c76b72015-02-23 11:09:24 -0800294 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700295 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall58c76b72015-02-23 11:09:24 -0800296 roleCheck = roleCheck and main.TRUE
297 else:
298 roleCheck = roleCheck and main.FALSE
299
300 # Assign switch
301 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
302 assert deviceId, "No device id for s6 in ONOS"
303 roleCall = roleCall and main.ONOScli1.deviceRole(
304 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700305 ip )
Jon Hall58c76b72015-02-23 11:09:24 -0800306 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700307 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall58c76b72015-02-23 11:09:24 -0800308 roleCheck = roleCheck and main.TRUE
309 else:
310 roleCheck = roleCheck and main.FALSE
311
Jon Hall5cfd23c2015-03-19 11:40:57 -0700312 ip = nodes[ 3 ].ip_address # ONOS4
Jon Hall58c76b72015-02-23 11:09:24 -0800313 # Assign switch
314 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
315 assert deviceId, "No device id for s4 in ONOS"
316 roleCall = roleCall and main.ONOScli1.deviceRole(
317 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700318 ip )
Jon Hall58c76b72015-02-23 11:09:24 -0800319 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700320 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall58c76b72015-02-23 11:09:24 -0800321 roleCheck = roleCheck and main.TRUE
322 else:
323 roleCheck = roleCheck and main.FALSE
324
Jon Hall5cfd23c2015-03-19 11:40:57 -0700325 ip = nodes[ 4 ].ip_address # ONOS5
Jon Hall58c76b72015-02-23 11:09:24 -0800326 for i in range( 8, 18 ):
327 dpid = '3' + str( i ).zfill( 3 )
328 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
329 assert deviceId, "No device id for s%i in ONOS" % i
330 roleCall = roleCall and main.ONOScli1.deviceRole(
331 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700332 ip )
Jon Hall58c76b72015-02-23 11:09:24 -0800333 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700334 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall58c76b72015-02-23 11:09:24 -0800335 roleCheck = roleCheck and main.TRUE
336 else:
337 roleCheck = roleCheck and main.FALSE
338
Jon Hall5cfd23c2015-03-19 11:40:57 -0700339 ip = nodes[ 5 ].ip_address # ONOS6
Jon Hall58c76b72015-02-23 11:09:24 -0800340 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
341 assert deviceId, "No device id for s7 in ONOS"
342 roleCall = roleCall and main.ONOScli1.deviceRole(
343 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700344 ip )
Jon Hall58c76b72015-02-23 11:09:24 -0800345 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700346 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall58c76b72015-02-23 11:09:24 -0800347 roleCheck = roleCheck and main.TRUE
348 else:
349 roleCheck = roleCheck and main.FALSE
350
Jon Hall5cfd23c2015-03-19 11:40:57 -0700351 ip = nodes[ 6 ].ip_address # ONOS7
Jon Hall58c76b72015-02-23 11:09:24 -0800352 for i in range( 18, 28 ):
353 dpid = '6' + str( i ).zfill( 3 )
354 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
355 assert deviceId, "No device id for s%i in ONOS" % i
356 roleCall = roleCall and main.ONOScli1.deviceRole(
357 deviceId,
Jon Hall5cfd23c2015-03-19 11:40:57 -0700358 ip )
Jon Hall58c76b72015-02-23 11:09:24 -0800359 # Check assignment
Jon Hall5cfd23c2015-03-19 11:40:57 -0700360 if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
Jon Hall58c76b72015-02-23 11:09:24 -0800361 roleCheck = roleCheck and main.TRUE
362 else:
363 roleCheck = roleCheck and main.FALSE
364 except ( AttributeError, AssertionError ):
365 main.log.exception( "Something is wrong with ONOS device view" )
366 main.log.info( main.ONOScli1.devices() )
Jon Hall6aec96b2015-01-19 14:49:31 -0800367 utilities.assert_equals(
368 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800369 actual=roleCall,
Jon Hall6aec96b2015-01-19 14:49:31 -0800370 onpass="Re-assigned switch mastership to designated controller",
Jon Hall8f89dda2015-01-22 16:03:33 -0800371 onfail="Something wrong with deviceRole calls" )
Jon Hall94fd0472014-12-08 11:52:42 -0800372
Jon Hall6aec96b2015-01-19 14:49:31 -0800373 utilities.assert_equals(
374 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800375 actual=roleCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800376 onpass="Switches were successfully reassigned to designated " +
377 "controller",
378 onfail="Switches were not successfully reassigned" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800379 mastershipCheck = mastershipCheck and roleCall and roleCheck
380 utilities.assert_equals( expect=main.TRUE, actual=mastershipCheck,
Jon Hall21270ac2015-02-16 17:59:55 -0800381 onpass="Switch mastership correctly assigned",
382 onfail="Error in (re)assigning switch" +
383 " mastership" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800384
Jon Hall6aec96b2015-01-19 14:49:31 -0800385 def CASE3( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800386 """
387 Assign intents
Jon Hall73cf9cc2014-11-20 22:28:38 -0800388 """
Jon Hall6aec96b2015-01-19 14:49:31 -0800389 # FIXME: we must reinstall intents until we have a persistant
390 # datastore!
Jon Hall73cf9cc2014-11-20 22:28:38 -0800391 import time
Jon Hall58c76b72015-02-23 11:09:24 -0800392 import json
Jon Hall5cfd23c2015-03-19 11:40:57 -0700393 assert numControllers, "numControllers not defined"
394 assert main, "main not defined"
395 assert utilities.assert_equals, "utilities.assert_equals not defined"
396 assert CLIs, "CLIs not defined"
397 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -0800398 main.log.report( "Adding host intents" )
399 main.case( "Adding host Intents" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800400
Jon Hall8f89dda2015-01-22 16:03:33 -0800401 main.step( "Discovering Hosts( Via pingall for now )" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800402 # FIXME: Once we have a host discovery mechanism, use that instead
Jon Hall73cf9cc2014-11-20 22:28:38 -0800403
Jon Hall6aec96b2015-01-19 14:49:31 -0800404 # install onos-app-fwd
405 main.log.info( "Install reactive forwarding app" )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700406 appResults = main.TRUE
407 threads = []
408 for i in range( numControllers ):
409 t = main.Thread( target=CLIs[i].featureInstall,
410 name="featureInstall-" + str( i ),
411 args=["onos-app-fwd"] )
412 threads.append( t )
413 t.start()
414
415 for t in threads:
416 t.join()
417 appResults = appResults and t.result
Jon Hall94fd0472014-12-08 11:52:42 -0800418
Jon Hall6aec96b2015-01-19 14:49:31 -0800419 # REACTIVE FWD test
Jon Hall8f89dda2015-01-22 16:03:33 -0800420 pingResult = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -0700421 for i in range(2): # Retry if pingall fails first time
422 time1 = time.time()
423 pingResult = main.Mininet1.pingall()
424 utilities.assert_equals(
425 expect=main.TRUE,
426 actual=pingResult,
427 onpass="Reactive Pingall test passed",
428 onfail="Reactive Pingall failed, one or more ping pairs failed" )
429 time2 = time.time()
430 main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800431
Jon Hall6aec96b2015-01-19 14:49:31 -0800432 # uninstall onos-app-fwd
433 main.log.info( "Uninstall reactive forwarding app" )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700434 threads = []
435 for i in range( numControllers ):
436 t = main.Thread( target=CLIs[i].featureUninstall,
437 name="featureUninstall-" + str( i ),
438 args=["onos-app-fwd"] )
439 threads.append( t )
440 t.start()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800441
Jon Hall5cfd23c2015-03-19 11:40:57 -0700442 for t in threads:
443 t.join()
444 appResults = appResults and t.result
445
446 # timeout for fwd flows
447 time.sleep( 11 )
448
449 main.step( "Add host intents" )
Jon Hall58c76b72015-02-23 11:09:24 -0800450 intentIds = []
Jon Hall6aec96b2015-01-19 14:49:31 -0800451 # TODO: move the host numbers to params
Jon Hall58c76b72015-02-23 11:09:24 -0800452 # Maybe look at all the paths we ping?
Jon Hall8f89dda2015-01-22 16:03:33 -0800453 intentAddResult = True
Jon Hall58c76b72015-02-23 11:09:24 -0800454 hostResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800455 for i in range( 8, 18 ):
456 main.log.info( "Adding host intent between h" + str( i ) +
457 " and h" + str( i + 10 ) )
458 host1 = "00:00:00:00:00:" + \
459 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
460 host2 = "00:00:00:00:00:" + \
461 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800462 # NOTE: getHost can return None
463 host1Dict = main.ONOScli1.getHost( host1 )
464 host2Dict = main.ONOScli1.getHost( host2 )
465 host1Id = None
466 host2Id = None
467 if host1Dict and host2Dict:
468 host1Id = host1Dict.get( 'id', None )
469 host2Id = host2Dict.get( 'id', None )
Jon Hall8f89dda2015-01-22 16:03:33 -0800470 if host1Id and host2Id:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700471 nodeNum = ( i % 7 )
472 tmpId = CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall63604932015-02-26 17:09:50 -0800473 if tmpId:
474 main.log.info( "Added intent with id: " + tmpId )
475 intentIds.append( tmpId )
476 else:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700477 main.log.error( "addHostIntent returned: " +
478 repr( tmpId ) )
Jon Hall669173b2014-12-17 11:36:30 -0800479 else:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700480 main.log.error( "Error, getHost() failed for h" + str( i ) +
481 " and/or h" + str( i + 10 ) )
482 hosts = CLIs[ 0 ].hosts()
483 main.log.warn( "Hosts output: " )
484 try:
485 main.log.warn( json.dumps( json.loads( hosts ),
486 sort_keys=True,
487 indent=4,
488 separators=( ',', ': ' ) ) )
489 except ( ValueError, TypeError ):
490 main.log.warn( repr( hosts ) )
Jon Hall58c76b72015-02-23 11:09:24 -0800491 hostResult = main.FALSE
492 onosIds = main.ONOScli1.getAllIntentsId()
493 main.log.info( "Submitted intents: " + str( intentIds ) )
494 main.log.info( "Intents in ONOS: " + str( onosIds ) )
495 for intent in intentIds:
496 if intent in onosIds:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700497 pass # intent submitted is in onos
Jon Hall58c76b72015-02-23 11:09:24 -0800498 else:
499 intentAddResult = False
Jon Hall1b8f54a2015-02-04 13:24:20 -0800500 # Print the intent states
Jon Hall58c76b72015-02-23 11:09:24 -0800501 intents = main.ONOScli1.intents()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800502 intentStates = []
Jon Hall63604932015-02-26 17:09:50 -0800503 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800504 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
505 count = 0
Jon Hall5cfd23c2015-03-19 11:40:57 -0700506 try:
507 for intent in json.loads( intents ):
508 state = intent.get( 'state', None )
509 if "INSTALLED" not in state:
510 installedCheck = False
511 intentId = intent.get( 'id', None )
512 intentStates.append( ( intentId, state ) )
513 except ( ValueError, TypeError ):
514 main.log.exception( "Error parsing intents" )
Jon Hall58c76b72015-02-23 11:09:24 -0800515 # add submitted intents not in the store
516 tmplist = [ i for i, s in intentStates ]
517 missingIntents = False
518 for i in intentIds:
519 if i not in tmplist:
520 intentStates.append( ( i, " - " ) )
521 missingIntents = True
522 intentStates.sort()
523 for i, s in intentStates:
524 count += 1
525 main.log.info( "%-6s%-15s%-15s" %
526 ( str( count ), str( i ), str( s ) ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700527 leaders = main.ONOScli1.leaders()
528 try:
529 if leaders:
530 parsedLeaders = json.loads( leaders )
531 main.log.warn( json.dumps( parsedLeaders,
532 sort_keys=True,
533 indent=4,
534 separators=( ',', ': ' ) ) )
535 # check for all intent partitions
536 # check for election
537 topics = []
538 for i in range( 14 ):
539 topics.append( "intent-partition-" + str( i ) )
540 # FIXME: this should only be after we start the app
541 topics.append( "org.onosproject.election" )
542 main.log.debug( topics )
543 ONOStopics = [ j['topic'] for j in parsedLeaders ]
544 for topic in topics:
545 if topic not in ONOStopics:
546 main.log.error( "Error: " + topic +
547 " not in leaders" )
548 else:
549 main.log.error( "leaders() returned None" )
550 except ( ValueError, TypeError ):
551 main.log.exception( "Error parsing leaders" )
552 main.log.error( repr( leaders ) )
553 partitions = main.ONOScli1.partitions()
554 try:
555 if partitions :
556 parsedPartitions = json.loads( partitions )
557 main.log.warn( json.dumps( parsedPartitions,
558 sort_keys=True,
559 indent=4,
560 separators=( ',', ': ' ) ) )
561 # TODO check for a leader in all paritions
562 # TODO check for consistency among nodes
563 else:
564 main.log.error( "partitions() returned None" )
565 except ( ValueError, TypeError ):
566 main.log.exception( "Error parsing partitions" )
567 main.log.error( repr( partitions ) )
Jon Hall63604932015-02-26 17:09:50 -0800568 pendingMap = main.ONOScli1.pendingMap()
Jon Hall5cfd23c2015-03-19 11:40:57 -0700569 try:
570 if pendingMap :
571 parsedPending = json.loads( pendingMap )
572 main.log.warn( json.dumps( parsedPending,
573 sort_keys=True,
574 indent=4,
575 separators=( ',', ': ' ) ) )
576 # TODO check something here?
577 else:
578 main.log.error( "pendingMap() returned None" )
579 except ( ValueError, TypeError ):
580 main.log.exception( "Error parsing pending map" )
581 main.log.error( repr( pendingMap ) )
582
Jon Hall58c76b72015-02-23 11:09:24 -0800583 intentAddResult = bool( pingResult and hostResult and intentAddResult
Jon Hall63604932015-02-26 17:09:50 -0800584 and not missingIntents and installedCheck )
Jon Hall6aec96b2015-01-19 14:49:31 -0800585 utilities.assert_equals(
586 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800587 actual=intentAddResult,
Jon Hall529a37f2015-01-28 10:02:00 -0800588 onpass="Pushed host intents to ONOS",
589 onfail="Error in pushing host intents to ONOS" )
Jon Hall58c76b72015-02-23 11:09:24 -0800590
Jon Hall63604932015-02-26 17:09:50 -0800591 if not intentAddResult or "key" in pendingMap:
Jon Hall58c76b72015-02-23 11:09:24 -0800592 import time
Jon Hall63604932015-02-26 17:09:50 -0800593 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800594 main.log.info( "Sleeping 60 seconds to see if intents are found" )
595 time.sleep( 60 )
596 onosIds = main.ONOScli1.getAllIntentsId()
597 main.log.info( "Submitted intents: " + str( intentIds ) )
598 main.log.info( "Intents in ONOS: " + str( onosIds ) )
599 # Print the intent states
600 intents = main.ONOScli1.intents()
601 intentStates = []
602 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
603 count = 0
Jon Hall5cfd23c2015-03-19 11:40:57 -0700604 try:
605 for intent in json.loads( intents ):
606 # Iter through intents of a node
607 state = intent.get( 'state', None )
608 if "INSTALLED" not in state:
609 installedCheck = False
610 intentId = intent.get( 'id', None )
611 intentStates.append( ( intentId, state ) )
612 except ( ValueError, TypeError ):
613 main.log.exception( "Error parsing intents" )
Jon Hall58c76b72015-02-23 11:09:24 -0800614 # add submitted intents not in the store
615 tmplist = [ i for i, s in intentStates ]
616 for i in intentIds:
617 if i not in tmplist:
618 intentStates.append( ( i, " - " ) )
619 intentStates.sort()
620 for i, s in intentStates:
621 count += 1
622 main.log.info( "%-6s%-15s%-15s" %
623 ( str( count ), str( i ), str( s ) ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700624 leaders = main.ONOScli1.leaders()
625 try:
626 if leaders:
627 parsedLeaders = json.loads( leaders )
628 main.log.warn( json.dumps( parsedLeaders,
629 sort_keys=True,
630 indent=4,
631 separators=( ',', ': ' ) ) )
632 # check for all intent partitions
633 # check for election
634 topics = []
635 for i in range( 14 ):
636 topics.append( "intent-partition-" + str( i ) )
637 # FIXME: this should only be after we start the app
638 topics.append( "org.onosproject.election" )
639 main.log.debug( topics )
640 ONOStopics = [ j['topic'] for j in parsedLeaders ]
641 for topic in topics:
642 if topic not in ONOStopics:
643 main.log.error( "Error: " + topic +
644 " not in leaders" )
645 else:
646 main.log.error( "leaders() returned None" )
647 except ( ValueError, TypeError ):
648 main.log.exception( "Error parsing leaders" )
649 main.log.error( repr( leaders ) )
650 partitions = main.ONOScli1.partitions()
651 try:
652 if partitions :
653 parsedPartitions = json.loads( partitions )
654 main.log.warn( json.dumps( parsedPartitions,
655 sort_keys=True,
656 indent=4,
657 separators=( ',', ': ' ) ) )
658 # TODO check for a leader in all paritions
659 # TODO check for consistency among nodes
660 else:
661 main.log.error( "partitions() returned None" )
662 except ( ValueError, TypeError ):
663 main.log.exception( "Error parsing partitions" )
664 main.log.error( repr( partitions ) )
665 pendingMap = main.ONOScli1.pendingMap()
666 try:
667 if pendingMap :
668 parsedPending = json.loads( pendingMap )
669 main.log.warn( json.dumps( parsedPending,
670 sort_keys=True,
671 indent=4,
672 separators=( ',', ': ' ) ) )
673 # TODO check something here?
674 else:
675 main.log.error( "pendingMap() returned None" )
676 except ( ValueError, TypeError ):
677 main.log.exception( "Error parsing pending map" )
678 main.log.error( repr( pendingMap ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800679
Jon Hall6aec96b2015-01-19 14:49:31 -0800680 def CASE4( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800681 """
682 Ping across added host intents
683 """
Jon Hall58c76b72015-02-23 11:09:24 -0800684 import json
Jon Hall5cfd23c2015-03-19 11:40:57 -0700685 import time
686 assert numControllers, "numControllers not defined"
687 assert main, "main not defined"
688 assert utilities.assert_equals, "utilities.assert_equals not defined"
689 assert CLIs, "CLIs not defined"
690 assert nodes, "nodes not defined"
Jon Hall73cf9cc2014-11-20 22:28:38 -0800691 description = " Ping across added host intents"
Jon Hall6aec96b2015-01-19 14:49:31 -0800692 main.log.report( description )
693 main.case( description )
Jon Hall8f89dda2015-01-22 16:03:33 -0800694 PingResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800695 for i in range( 8, 18 ):
Jon Hall58c76b72015-02-23 11:09:24 -0800696 ping = main.Mininet1.pingHost( src="h" + str( i ),
697 target="h" + str( i + 10 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800698 PingResult = PingResult and ping
Jon Hall6aec96b2015-01-19 14:49:31 -0800699 if ping == main.FALSE:
700 main.log.warn( "Ping failed between h" + str( i ) +
701 " and h" + str( i + 10 ) )
702 elif ping == main.TRUE:
703 main.log.info( "Ping test passed!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800704 # Don't set PingResult or you'd override failures
Jon Hall8f89dda2015-01-22 16:03:33 -0800705 if PingResult == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800706 main.log.report(
707 "Intents have not been installed correctly, pings failed." )
Jon Hall58c76b72015-02-23 11:09:24 -0800708 # TODO: pretty print
Jon Hall5cfd23c2015-03-19 11:40:57 -0700709 main.log.warn( "ONOS1 intents: " )
710 try:
711 tmpIntents = main.ONOScli1.intents()
712 main.log.warn( json.dumps( json.loads( tmpIntents ),
713 sort_keys=True,
714 indent=4,
715 separators=( ',', ': ' ) ) )
716 except ( ValueError, TypeError ):
717 main.log.warn( repr( tmpIntents ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800718 if PingResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800719 main.log.report(
720 "Intents have been installed correctly and verified by pings" )
721 utilities.assert_equals(
722 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800723 actual=PingResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800724 onpass="Intents have been installed correctly and pings work",
725 onfail="Intents have not been installed correctly, pings failed." )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800726
Jon Hall63604932015-02-26 17:09:50 -0800727 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800728 if PingResult is not main.TRUE:
729 # Print the intent states
730 intents = main.ONOScli1.intents()
731 intentStates = []
732 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
733 count = 0
734 # Iter through intents of a node
Jon Hall5cfd23c2015-03-19 11:40:57 -0700735 try:
736 for intent in json.loads( intents ):
737 state = intent.get( 'state', None )
738 if "INSTALLED" not in state:
739 installedCheck = False
740 intentId = intent.get( 'id', None )
741 intentStates.append( ( intentId, state ) )
742 except ( ValueError, TypeError ):
743 main.log.exception( "Error parsing intents." )
Jon Hall58c76b72015-02-23 11:09:24 -0800744 intentStates.sort()
745 for i, s in intentStates:
746 count += 1
747 main.log.info( "%-6s%-15s%-15s" %
748 ( str( count ), str( i ), str( s ) ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700749 leaders = main.ONOScli1.leaders()
750 try:
751 if leaders:
752 parsedLeaders = json.loads( leaders )
753 main.log.warn( json.dumps( parsedLeaders,
754 sort_keys=True,
755 indent=4,
756 separators=( ',', ': ' ) ) )
757 # check for all intent partitions
758 # check for election
759 topics = []
760 for i in range( 14 ):
761 topics.append( "intent-partition-" + str( i ) )
762 # FIXME: this should only be after we start the app
763 topics.append( "org.onosproject.election" )
764 main.log.debug( topics )
765 ONOStopics = [ j['topic'] for j in parsedLeaders ]
766 for topic in topics:
767 if topic not in ONOStopics:
768 main.log.error( "Error: " + topic +
769 " not in leaders" )
770 else:
771 main.log.error( "leaders() returned None" )
772 except ( ValueError, TypeError ):
773 main.log.exception( "Error parsing leaders" )
774 main.log.error( repr( leaders ) )
775 partitions = main.ONOScli1.partitions()
776 try:
777 if partitions :
778 parsedPartitions = json.loads( partitions )
779 main.log.warn( json.dumps( parsedPartitions,
780 sort_keys=True,
781 indent=4,
782 separators=( ',', ': ' ) ) )
783 # TODO check for a leader in all paritions
784 # TODO check for consistency among nodes
785 else:
786 main.log.error( "partitions() returned None" )
787 except ( ValueError, TypeError ):
788 main.log.exception( "Error parsing partitions" )
789 main.log.error( repr( partitions ) )
790 pendingMap = main.ONOScli1.pendingMap()
791 try:
792 if pendingMap :
793 parsedPending = json.loads( pendingMap )
794 main.log.warn( json.dumps( parsedPending,
795 sort_keys=True,
796 indent=4,
797 separators=( ',', ': ' ) ) )
798 # TODO check something here?
799 else:
800 main.log.error( "pendingMap() returned None" )
801 except ( ValueError, TypeError ):
802 main.log.exception( "Error parsing pending map" )
803 main.log.error( repr( pendingMap ) )
804
Jon Hall63604932015-02-26 17:09:50 -0800805 if not installedCheck:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700806 main.log.info( "Waiting 60 seconds to see if the state of " +
807 "intents change" )
Jon Hall63604932015-02-26 17:09:50 -0800808 time.sleep( 60 )
809 # Print the intent states
810 intents = main.ONOScli1.intents()
811 intentStates = []
812 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
813 count = 0
814 # Iter through intents of a node
Jon Hall5cfd23c2015-03-19 11:40:57 -0700815 try:
816 for intent in json.loads( intents ):
817 state = intent.get( 'state', None )
818 if "INSTALLED" not in state:
819 installedCheck = False
820 intentId = intent.get( 'id', None )
821 intentStates.append( ( intentId, state ) )
822 except ( ValueError, TypeError ):
823 main.log.exception( "Error parsing intents." )
Jon Hall63604932015-02-26 17:09:50 -0800824 intentStates.sort()
825 for i, s in intentStates:
826 count += 1
827 main.log.info( "%-6s%-15s%-15s" %
828 ( str( count ), str( i ), str( s ) ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700829 leaders = main.ONOScli1.leaders()
830 try:
831 if leaders:
832 parsedLeaders = json.loads( leaders )
833 main.log.warn( json.dumps( parsedLeaders,
834 sort_keys=True,
835 indent=4,
836 separators=( ',', ': ' ) ) )
837 # check for all intent partitions
838 # check for election
839 topics = []
840 for i in range( 14 ):
841 topics.append( "intent-partition-" + str( i ) )
842 # FIXME: this should only be after we start the app
843 topics.append( "org.onosproject.election" )
844 main.log.debug( topics )
845 ONOStopics = [ j['topic'] for j in parsedLeaders ]
846 for topic in topics:
847 if topic not in ONOStopics:
848 main.log.error( "Error: " + topic +
849 " not in leaders" )
850 else:
851 main.log.error( "leaders() returned None" )
852 except ( ValueError, TypeError ):
853 main.log.exception( "Error parsing leaders" )
854 main.log.error( repr( leaders ) )
855 partitions = main.ONOScli1.partitions()
856 try:
857 if partitions :
858 parsedPartitions = json.loads( partitions )
859 main.log.warn( json.dumps( parsedPartitions,
860 sort_keys=True,
861 indent=4,
862 separators=( ',', ': ' ) ) )
863 # TODO check for a leader in all paritions
864 # TODO check for consistency among nodes
865 else:
866 main.log.error( "partitions() returned None" )
867 except ( ValueError, TypeError ):
868 main.log.exception( "Error parsing partitions" )
869 main.log.error( repr( partitions ) )
870 pendingMap = main.ONOScli1.pendingMap()
871 try:
872 if pendingMap :
873 parsedPending = json.loads( pendingMap )
874 main.log.warn( json.dumps( parsedPending,
875 sort_keys=True,
876 indent=4,
877 separators=( ',', ': ' ) ) )
878 # TODO check something here?
879 else:
880 main.log.error( "pendingMap() returned None" )
881 except ( ValueError, TypeError ):
882 main.log.exception( "Error parsing pending map" )
883 main.log.error( repr( pendingMap ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800884
Jon Hall6aec96b2015-01-19 14:49:31 -0800885 def CASE5( self, main ):
886 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800887 Reading state of ONOS
Jon Hall6aec96b2015-01-19 14:49:31 -0800888 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800889 import json
Jon Hall5cfd23c2015-03-19 11:40:57 -0700890 import time
891 assert numControllers, "numControllers not defined"
892 assert main, "main not defined"
893 assert utilities.assert_equals, "utilities.assert_equals not defined"
894 assert CLIs, "CLIs not defined"
895 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -0800896 # assumes that sts is already in you PYTHONPATH
897 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -0800898
Jon Hall6aec96b2015-01-19 14:49:31 -0800899 main.log.report( "Setting up and gathering data for current state" )
900 main.case( "Setting up and gathering data for current state" )
901 # The general idea for this test case is to pull the state of
902 # ( intents,flows, topology,... ) from each ONOS node
Jon Hall5cfd23c2015-03-19 11:40:57 -0700903 # We can then compare them with each other and also with past states
Jon Hall73cf9cc2014-11-20 22:28:38 -0800904
Jon Hall5cfd23c2015-03-19 11:40:57 -0700905 main.step( "Check that each switch has a master" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800906 global mastershipState
Jon Hall5cfd23c2015-03-19 11:40:57 -0700907 mastershipState = '[]'
Jon Hall94fd0472014-12-08 11:52:42 -0800908
Jon Hall6aec96b2015-01-19 14:49:31 -0800909 # Assert that each device has a master
Jon Hall5cfd23c2015-03-19 11:40:57 -0700910 rolesNotNull = main.TRUE
911 threads = []
912 for i in range( numControllers ):
913 t = main.Thread( target=CLIs[i].rolesNotNull,
914 name="rolesNotNull-" + str( i ),
915 args=[] )
916 threads.append( t )
917 t.start()
918
919 for t in threads:
920 t.join()
921 rolesNotNull = rolesNotNull and t.result
Jon Hall6aec96b2015-01-19 14:49:31 -0800922 utilities.assert_equals(
923 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800924 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -0800925 onpass="Each device has a master",
926 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -0800927
Jon Hall5cfd23c2015-03-19 11:40:57 -0700928 main.step( "Get the Mastership of each switch from each controller" )
929 ONOSMastership = []
930 mastershipCheck = main.FALSE
931 consistentMastership = True
932 rolesResults = True
933 threads = []
934 for i in range( numControllers ):
935 t = main.Thread( target=CLIs[i].roles,
936 name="roles-" + str( i ),
937 args=[] )
938 threads.append( t )
939 t.start()
940
941 for t in threads:
942 t.join()
943 ONOSMastership.append( t.result )
944
945 for i in range( numControllers ):
946 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
947 main.log.report( "Error in getting ONOS" + str( i + 1 ) +
948 " roles" )
949 main.log.warn(
950 "ONOS" + str( i + 1 ) + " mastership response: " +
951 repr( ONOSMastership[i] ) )
952 rolesResults = False
953 utilities.assert_equals(
954 expect=True,
955 actual=rolesResults,
956 onpass="No error in reading roles output",
957 onfail="Error in reading roles from ONOS" )
958
959 main.step( "Check for consistency in roles from each controller" )
960 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall6aec96b2015-01-19 14:49:31 -0800961 main.log.report(
962 "Switch roles are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800963 else:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700964 consistentMastership = False
Jon Hall6aec96b2015-01-19 14:49:31 -0800965 utilities.assert_equals(
Jon Hall5cfd23c2015-03-19 11:40:57 -0700966 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800967 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -0800968 onpass="Switch roles are consistent across all ONOS nodes",
969 onfail="ONOS nodes have different views of switch roles" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800970
Jon Hall5cfd23c2015-03-19 11:40:57 -0700971 if rolesResults and not consistentMastership:
972 for i in range( numControllers ):
973 try:
974 main.log.warn(
975 "ONOS" + str( i + 1 ) + " roles: ",
976 json.dumps(
977 json.loads( ONOSMastership[ i ] ),
978 sort_keys=True,
979 indent=4,
980 separators=( ',', ': ' ) ) )
981 except ( ValueError, TypeError ):
982 main.log.warn( repr( ONOSMastership[ i ] ) )
983 elif rolesResults and consistentMastership:
984 mastershipCheck = main.TRUE
985 mastershipState = ONOSMastership[ 0 ]
986
Jon Hall6aec96b2015-01-19 14:49:31 -0800987 main.step( "Get the intents from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800988 global intentState
989 intentState = []
Jon Hall5cfd23c2015-03-19 11:40:57 -0700990 ONOSIntents = []
Jon Hall8f89dda2015-01-22 16:03:33 -0800991 intentCheck = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -0700992 consistentIntents = True
993 intentsResults = True
994 threads = []
995 for i in range( numControllers ):
996 t = main.Thread( target=CLIs[i].intents,
997 name="intents-" + str( i ),
998 args=[],
999 kwargs={ 'jsonFormat': True } )
1000 threads.append( t )
1001 t.start()
1002
1003 for t in threads:
1004 t.join()
1005 ONOSIntents.append( t.result )
1006
1007 for i in range( numControllers ):
1008 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1009 main.log.report( "Error in getting ONOS" + str( i + 1 ) +
1010 " intents" )
1011 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1012 repr( ONOSIntents[ i ] ) )
1013 intentsResults = False
Jon Hall6aec96b2015-01-19 14:49:31 -08001014 utilities.assert_equals(
Jon Hall5cfd23c2015-03-19 11:40:57 -07001015 expect=True,
1016 actual=intentsResults,
1017 onpass="No error in reading intents output",
1018 onfail="Error in reading intents from ONOS" )
1019
1020 main.step( "Check for consistency in Intents from each controller" )
1021 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1022 main.log.report( "Intents are consistent across all ONOS " +
1023 "nodes" )
1024 else:
1025 consistentIntents = False
1026 main.log.report( "Intents not consistent" )
1027 utilities.assert_equals(
1028 expect=True,
1029 actual=consistentIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001030 onpass="Intents are consistent across all ONOS nodes",
1031 onfail="ONOS nodes have different views of intents" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001032
Jon Hall5cfd23c2015-03-19 11:40:57 -07001033 if intentsResults and not consistentIntents:
1034 n = len(ONOSIntents)
1035 main.log.warn( "ONOS" + str( n ) + " intents: " )
1036 main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1037 sort_keys=True,
1038 indent=4,
1039 separators=( ',', ': ' ) ) )
1040 for i in range( numControllers ):
1041 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1042 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1043 main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
1044 sort_keys=True,
1045 indent=4,
1046 separators=( ',', ': ' ) ) )
1047 else:
1048 main.log.warn( nodes[ i ].name + " intents match ONOS" +
1049 str( n ) + " intents" )
1050 elif intentsResults and consistentIntents:
1051 intentCheck = main.TRUE
1052 intentState = ONOSIntents[ 0 ]
1053
Jon Hall6aec96b2015-01-19 14:49:31 -08001054 main.step( "Get the flows from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001055 global flowState
1056 flowState = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001057 ONOSFlows = []
1058 ONOSFlowsJson = []
Jon Hall8f89dda2015-01-22 16:03:33 -08001059 flowCheck = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -07001060 consistentFlows = True
1061 flowsResults = True
1062 threads = []
1063 for i in range( numControllers ):
1064 t = main.Thread( target=CLIs[i].flows,
1065 name="flows-" + str( i ),
1066 args=[],
1067 kwargs={ 'jsonFormat': True } )
1068 threads.append( t )
1069 t.start()
1070
1071 time.sleep(30)
1072 for t in threads:
1073 t.join()
1074 result = t.result
1075 ONOSFlows.append( result )
1076
1077 for i in range( numControllers ):
1078 num = str( i + 1 )
1079 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1080 main.log.report( "Error in getting ONOS" + num + " flows" )
1081 main.log.warn( "ONOS" + num + " flows response: " +
1082 repr( ONOSFlows[ i ] ) )
1083 flowsResults = False
1084 ONOSFlowsJson.append( None )
Jon Hall58c76b72015-02-23 11:09:24 -08001085 else:
Jon Hall5cfd23c2015-03-19 11:40:57 -07001086 try:
1087 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1088 except ( ValueError, TypeError ):
1089 # FIXME: change this to log.error?
1090 main.log.exception( "Error in parsing ONOS" + num +
1091 " response as json." )
1092 main.log.error( repr( ONOSFlows[ i ] ) )
1093 ONOSFlowsJson.append( None )
1094 flowsResults = False
Jon Hall6aec96b2015-01-19 14:49:31 -08001095 utilities.assert_equals(
Jon Hall5cfd23c2015-03-19 11:40:57 -07001096 expect=True,
1097 actual=flowsResults,
1098 onpass="No error in reading flows output",
1099 onfail="Error in reading flows from ONOS" )
1100
1101 main.step( "Check for consistency in Flows from each controller" )
1102 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1103 if all( tmp ):
1104 main.log.report( "Flow count is consistent across all ONOS nodes" )
1105 else:
1106 consistentFlows = False
1107 utilities.assert_equals(
1108 expect=True,
1109 actual=consistentFlows,
Jon Hall6aec96b2015-01-19 14:49:31 -08001110 onpass="The flow count is consistent across all ONOS nodes",
1111 onfail="ONOS nodes have different flow counts" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001112
Jon Hall5cfd23c2015-03-19 11:40:57 -07001113 if flowsResults and not consistentFlows:
1114 for i in range( numControllers ):
1115 try:
1116 main.log.warn(
1117 "ONOS" + str( i + 1 ) + " flows: " +
1118 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1119 indent=4, separators=( ',', ': ' ) ) )
1120 except ( ValueError, TypeError ):
1121 main.log.warn(
1122 "ONOS" + str( i + 1 ) + " flows: " +
1123 repr( ONOSFlows[ i ] ) )
1124 elif flowsResults and consistentFlows:
1125 flowCheck = main.TRUE
1126 flowState = ONOSFlows[ 0 ]
1127
Jon Hall6aec96b2015-01-19 14:49:31 -08001128 main.step( "Get the OF Table entries" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001129 global flows
Jon Hall6aec96b2015-01-19 14:49:31 -08001130 flows = []
1131 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001132 flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001133 if flowCheck == main.FALSE:
1134 for table in flows:
1135 main.log.warn( table )
Jon Hall6aec96b2015-01-19 14:49:31 -08001136 # TODO: Compare switch flow tables with ONOS flow tables
Jon Hall73cf9cc2014-11-20 22:28:38 -08001137
Jon Hall6aec96b2015-01-19 14:49:31 -08001138 main.step( "Start continuous pings" )
1139 main.Mininet2.pingLong(
1140 src=main.params[ 'PING' ][ 'source1' ],
1141 target=main.params[ 'PING' ][ 'target1' ],
1142 pingTime=500 )
1143 main.Mininet2.pingLong(
1144 src=main.params[ 'PING' ][ 'source2' ],
1145 target=main.params[ 'PING' ][ 'target2' ],
1146 pingTime=500 )
1147 main.Mininet2.pingLong(
1148 src=main.params[ 'PING' ][ 'source3' ],
1149 target=main.params[ 'PING' ][ 'target3' ],
1150 pingTime=500 )
1151 main.Mininet2.pingLong(
1152 src=main.params[ 'PING' ][ 'source4' ],
1153 target=main.params[ 'PING' ][ 'target4' ],
1154 pingTime=500 )
1155 main.Mininet2.pingLong(
1156 src=main.params[ 'PING' ][ 'source5' ],
1157 target=main.params[ 'PING' ][ 'target5' ],
1158 pingTime=500 )
1159 main.Mininet2.pingLong(
1160 src=main.params[ 'PING' ][ 'source6' ],
1161 target=main.params[ 'PING' ][ 'target6' ],
1162 pingTime=500 )
1163 main.Mininet2.pingLong(
1164 src=main.params[ 'PING' ][ 'source7' ],
1165 target=main.params[ 'PING' ][ 'target7' ],
1166 pingTime=500 )
1167 main.Mininet2.pingLong(
1168 src=main.params[ 'PING' ][ 'source8' ],
1169 target=main.params[ 'PING' ][ 'target8' ],
1170 pingTime=500 )
1171 main.Mininet2.pingLong(
1172 src=main.params[ 'PING' ][ 'source9' ],
1173 target=main.params[ 'PING' ][ 'target9' ],
1174 pingTime=500 )
1175 main.Mininet2.pingLong(
1176 src=main.params[ 'PING' ][ 'source10' ],
1177 target=main.params[ 'PING' ][ 'target10' ],
1178 pingTime=500 )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001179
Jon Hall6aec96b2015-01-19 14:49:31 -08001180 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001181 ctrls = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001182 for node in nodes:
1183 temp = ( node, node.name, node.ip_address, 6633 )
1184 ctrls.append( temp )
1185 MNTopo = TestONTopology( main.Mininet1, ctrls )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001186
Jon Hall6aec96b2015-01-19 14:49:31 -08001187 main.step( "Collecting topology information from ONOS" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001188 devices = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001189 threads = []
1190 for i in range( numControllers ):
1191 t = main.Thread( target=CLIs[i].devices,
1192 name="devices-" + str( i ),
1193 args=[ ] )
1194 threads.append( t )
1195 t.start()
1196
1197 for t in threads:
1198 t.join()
1199 devices.append( t.result )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001200 hosts = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001201 threads = []
1202 for i in range( numControllers ):
1203 t = main.Thread( target=CLIs[i].hosts,
1204 name="hosts-" + str( i ),
1205 args=[ ] )
1206 threads.append( t )
1207 t.start()
1208
1209 for t in threads:
1210 t.join()
1211 try:
1212 hosts.append( json.loads( t.result ) )
1213 except ( ValueError, TypeError ):
1214 # FIXME: better handling of this, print which node
1215 # Maybe use thread name?
1216 main.log.exception( "Error parsing json output of hosts" )
1217 # FIXME: should this be an empty json object instead?
1218 hosts.append( None )
1219
Jon Hall73cf9cc2014-11-20 22:28:38 -08001220 ports = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001221 threads = []
1222 for i in range( numControllers ):
1223 t = main.Thread( target=CLIs[i].ports,
1224 name="ports-" + str( i ),
1225 args=[ ] )
1226 threads.append( t )
1227 t.start()
1228
1229 for t in threads:
1230 t.join()
1231 ports.append( t.result )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001232 links = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001233 threads = []
1234 for i in range( numControllers ):
1235 t = main.Thread( target=CLIs[i].links,
1236 name="links-" + str( i ),
1237 args=[ ] )
1238 threads.append( t )
1239 t.start()
1240
1241 for t in threads:
1242 t.join()
1243 links.append( t.result )
Jon Hall94fd0472014-12-08 11:52:42 -08001244 clusters = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001245 threads = []
1246 for i in range( numControllers ):
1247 t = main.Thread( target=CLIs[i].clusters,
1248 name="clusters-" + str( i ),
1249 args=[ ] )
1250 threads.append( t )
1251 t.start()
1252
1253 for t in threads:
1254 t.join()
1255 clusters.append( t.result )
Jon Hall529a37f2015-01-28 10:02:00 -08001256 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001257
Jon Hall6aec96b2015-01-19 14:49:31 -08001258 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -08001259 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001260 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001261 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001262 if "Error" not in hosts[ controller ]:
1263 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001264 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001265 else: # hosts not consistent
1266 main.log.report( "hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001267 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001268 " is inconsistent with ONOS1" )
1269 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001270 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001271
1272 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001273 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001274 controllerStr )
1275 consistentHostsResult = main.FALSE
1276 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001277 " hosts response: " +
1278 repr( hosts[ controller ] ) )
1279 utilities.assert_equals(
1280 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001281 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001282 onpass="Hosts view is consistent across all ONOS nodes",
1283 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001284
Jon Hall58c76b72015-02-23 11:09:24 -08001285 ipResult = main.TRUE
1286 for controller in range( 0, len( hosts ) ):
1287 controllerStr = str( controller + 1 )
1288 for host in hosts[ controller ]:
Jon Hall5cfd23c2015-03-19 11:40:57 -07001289 if not host.get( 'ips', [ ] ):
1290 main.log.error( "DEBUG:Error with host ips on controller" +
1291 controllerStr + ": " + str( host ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001292 ipResult = main.FALSE
1293 utilities.assert_equals(
1294 expect=main.TRUE,
1295 actual=ipResult,
1296 onpass="The ips of the hosts aren't empty",
1297 onfail="The ip of at least one host is missing" )
1298
Jon Hall6aec96b2015-01-19 14:49:31 -08001299 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001300 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001301 for controller in range( len( clusters ) ):
Jon Hall5cfd23c2015-03-19 11:40:57 -07001302 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001303 if "Error" not in clusters[ controller ]:
1304 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001305 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001306 else: # clusters not consistent
Jon Hall5cfd23c2015-03-19 11:40:57 -07001307 main.log.report( "clusters from ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001308 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001309 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001310
1311 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001312 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001313 "from ONOS" + controllerStr )
1314 consistentClustersResult = main.FALSE
1315 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001316 " clusters response: " +
1317 repr( clusters[ controller ] ) )
1318 utilities.assert_equals(
1319 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001320 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001321 onpass="Clusters view is consistent across all ONOS nodes",
1322 onfail="ONOS nodes have different views of clusters" )
1323 # there should always only be one cluster
Jon Hall5cfd23c2015-03-19 11:40:57 -07001324 try:
1325 numClusters = len( json.loads( clusters[ 0 ] ) )
1326 except ( ValueError, TypeError ):
1327 main.log.exception( "Error parsing clusters[0]: " +
1328 repr( clusters[ 0 ] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001329 clusterResults = main.FALSE
1330 if numClusters == 1:
1331 clusterResults = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001332 utilities.assert_equals(
1333 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001334 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001335 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08001336 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001337
Jon Hall6aec96b2015-01-19 14:49:31 -08001338 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001339 devicesResults = main.TRUE
1340 portsResults = main.TRUE
1341 linksResults = main.TRUE
1342 for controller in range( numControllers ):
1343 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001344 if devices[ controller ] or "Error" not in devices[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001345 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001346 MNTopo,
Jon Hall5cfd23c2015-03-19 11:40:57 -07001347 json.loads( devices[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001348 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001349 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001350 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001351 actual=currentDevicesResult,
1352 onpass="ONOS" + controllerStr +
1353 " Switches view is correct",
1354 onfail="ONOS" + controllerStr +
1355 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001356
Jon Hall6aec96b2015-01-19 14:49:31 -08001357 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001358 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001359 MNTopo,
Jon Hall5cfd23c2015-03-19 11:40:57 -07001360 json.loads( ports[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001361 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001362 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001363 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001364 actual=currentPortsResult,
1365 onpass="ONOS" + controllerStr +
1366 " ports view is correct",
1367 onfail="ONOS" + controllerStr +
1368 " ports view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001369
Jon Hall6aec96b2015-01-19 14:49:31 -08001370 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001371 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001372 MNTopo,
Jon Hall5cfd23c2015-03-19 11:40:57 -07001373 json.loads( links[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001374 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001375 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001376 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001377 actual=currentLinksResult,
1378 onpass="ONOS" + controllerStr +
1379 " links view is correct",
1380 onfail="ONOS" + controllerStr +
1381 " links view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001382
Jon Hall8f89dda2015-01-22 16:03:33 -08001383 devicesResults = devicesResults and currentDevicesResult
1384 portsResults = portsResults and currentPortsResult
1385 linksResults = linksResults and currentLinksResult
Jon Hall73cf9cc2014-11-20 22:28:38 -08001386
Jon Hall5cfd23c2015-03-19 11:40:57 -07001387 topoResult = ( devicesResults and portsResults and linksResults
1388 and consistentHostsResult and consistentClustersResult
1389 and clusterResults and ipResult )
Jon Hall8f89dda2015-01-22 16:03:33 -08001390 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08001391 onpass="Topology Check Test successful",
1392 onfail="Topology Check Test NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001393
Jon Hall8f89dda2015-01-22 16:03:33 -08001394 finalAssert = main.TRUE
Jon Hall5cfd23c2015-03-19 11:40:57 -07001395 finalAssert = ( finalAssert and topoResult and flowCheck
1396 and intentCheck and consistentMastership
1397 and rolesNotNull )
Jon Hall8f89dda2015-01-22 16:03:33 -08001398 utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
Jon Hall58c76b72015-02-23 11:09:24 -08001399 onpass="State check successful",
1400 onfail="State check NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001401
Jon Hall6aec96b2015-01-19 14:49:31 -08001402 def CASE6( self, main ):
1403 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001404 The Failure case.
Jon Hall6aec96b2015-01-19 14:49:31 -08001405 """
Jon Hall5cfd23c2015-03-19 11:40:57 -07001406 assert numControllers, "numControllers not defined"
1407 assert main, "main not defined"
1408 assert utilities.assert_equals, "utilities.assert_equals not defined"
1409 assert CLIs, "CLIs not defined"
1410 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08001411 main.log.report( "Restart entire ONOS cluster" )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001412 main.case( "Restart entire ONOS cluster" )
1413 main.step( "Killing ONOS nodes" )
1414 killResults = main.TRUE
1415 for node in nodes:
1416 killed = main.ONOSbench.onosKill( node.ip_address )
1417 killResults = killResults and killed
Jon Hall73cf9cc2014-11-20 22:28:38 -08001418
Jon Hall6aec96b2015-01-19 14:49:31 -08001419 main.step( "Checking if ONOS is up yet" )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001420 for i in range( 2 ):
1421 onosIsupResult = main.TRUE
1422 for node in nodes:
1423 started = main.ONOSbench.isup( node.ip_address )
1424 if not started:
1425 main.log.report( node.name + " didn't start!" )
1426 onosIsupResult = onosIsupResult and started
1427 if onosIsupResult == main.TRUE:
1428 break
Jon Hall73cf9cc2014-11-20 22:28:38 -08001429
Jon Hall5cfd23c2015-03-19 11:40:57 -07001430 main.log.step( "Starting ONOS CLI sessions" )
1431 cliResults = main.TRUE
1432 threads = []
1433 for i in range( numControllers ):
1434 t = main.Thread( target=CLIs[i].startOnosCli,
1435 name="startOnosCli-" + str( i ),
1436 args=[nodes[i].ip_address] )
1437 threads.append( t )
1438 t.start()
1439
1440 for t in threads:
1441 t.join()
1442 cliResults = cliResults and t.result
Jon Hall73cf9cc2014-11-20 22:28:38 -08001443
Jon Hall8f89dda2015-01-22 16:03:33 -08001444 caseResults = main.TRUE and onosIsupResult and cliResults
1445 utilities.assert_equals( expect=main.TRUE, actual=caseResults,
Jon Hall58c76b72015-02-23 11:09:24 -08001446 onpass="ONOS restart successful",
1447 onfail="ONOS restart NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001448
Jon Hall6aec96b2015-01-19 14:49:31 -08001449 def CASE7( self, main ):
1450 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001451 Check state after ONOS failure
Jon Hall6aec96b2015-01-19 14:49:31 -08001452 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001453 import json
Jon Hall5cfd23c2015-03-19 11:40:57 -07001454 assert numControllers, "numControllers not defined"
1455 assert main, "main not defined"
1456 assert utilities.assert_equals, "utilities.assert_equals not defined"
1457 assert CLIs, "CLIs not defined"
1458 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08001459 main.case( "Running ONOS Constant State Tests" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001460
Jon Hall5cfd23c2015-03-19 11:40:57 -07001461 main.step( "Check that each switch has a master" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001462 # Assert that each device has a master
Jon Hall5cfd23c2015-03-19 11:40:57 -07001463 rolesNotNull = main.TRUE
1464 threads = []
1465 for i in range( numControllers ):
1466 t = main.Thread( target=CLIs[i].rolesNotNull,
1467 name="rolesNotNull-" + str( i ),
1468 args=[ ] )
1469 threads.append( t )
1470 t.start()
1471
1472 for t in threads:
1473 t.join()
1474 rolesNotNull = rolesNotNull and t.result
Jon Hall6aec96b2015-01-19 14:49:31 -08001475 utilities.assert_equals(
1476 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001477 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -08001478 onpass="Each device has a master",
1479 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -08001480
Jon Hall5cfd23c2015-03-19 11:40:57 -07001481 ONOSMastership = []
1482 mastershipCheck = main.FALSE
1483 consistentMastership = True
1484 rolesResults = True
1485 threads = []
1486 for i in range( numControllers ):
1487 t = main.Thread( target=CLIs[i].roles,
1488 name="roles-" + str( i ),
1489 args=[] )
1490 threads.append( t )
1491 t.start()
1492
1493 for t in threads:
1494 t.join()
1495 ONOSMastership.append( t.result )
1496
1497 for i in range( numControllers ):
1498 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1499 main.log.report( "Error in getting ONOS" + str( i + 1 ) +
1500 " roles" )
1501 main.log.warn(
1502 "ONOS" + str( i + 1 ) + " mastership response: " +
1503 repr( ONOSMastership[i] ) )
1504 rolesResults = False
1505 utilities.assert_equals(
1506 expect=True,
1507 actual=rolesResults,
1508 onpass="No error in reading roles output",
1509 onfail="Error in reading roles from ONOS" )
1510
1511 main.step( "Check for consistency in roles from each controller" )
1512 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001513 main.log.report(
1514 "Switch roles are consistent across all ONOS nodes" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001515 else:
Jon Hall5cfd23c2015-03-19 11:40:57 -07001516 consistentMastership = False
Jon Hall6aec96b2015-01-19 14:49:31 -08001517 utilities.assert_equals(
Jon Hall5cfd23c2015-03-19 11:40:57 -07001518 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -08001519 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -08001520 onpass="Switch roles are consistent across all ONOS nodes",
1521 onfail="ONOS nodes have different views of switch roles" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001522
Jon Hall5cfd23c2015-03-19 11:40:57 -07001523 if rolesResults and not consistentMastership:
1524 for i in range( numControllers ):
1525 main.log.warn(
1526 "ONOS" + str( i + 1 ) + " roles: ",
1527 json.dumps(
1528 json.loads( ONOSMastership[ i ] ),
1529 sort_keys=True,
1530 indent=4,
1531 separators=( ',', ': ' ) ) )
1532 elif rolesResults and not consistentMastership:
1533 mastershipCheck = main.TRUE
1534
Jon Hall73cf9cc2014-11-20 22:28:38 -08001535 description2 = "Compare switch roles from before failure"
Jon Hall6aec96b2015-01-19 14:49:31 -08001536 main.step( description2 )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001537 try:
1538 currentJson = json.loads( ONOSMastership[0] )
1539 oldJson = json.loads( mastershipState )
1540 except ( ValueError, TypeError ):
1541 main.log.exception( "Something is wrong with parsing " +
1542 "ONOSMastership[0] or mastershipState" )
1543 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1544 main.log.error( "mastershipState" + repr( mastershipState ) )
1545 main.cleanup()
1546 main.exit()
Jon Hall8f89dda2015-01-22 16:03:33 -08001547 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001548 for i in range( 1, 29 ):
1549 switchDPID = str(
Jon Hall5cfd23c2015-03-19 11:40:57 -07001550 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001551 current = [ switch[ 'master' ] for switch in currentJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001552 if switchDPID in switch[ 'id' ] ]
Jon Hall8f89dda2015-01-22 16:03:33 -08001553 old = [ switch[ 'master' ] for switch in oldJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001554 if switchDPID in switch[ 'id' ] ]
Jon Hall73cf9cc2014-11-20 22:28:38 -08001555 if current == old:
Jon Hall8f89dda2015-01-22 16:03:33 -08001556 mastershipCheck = mastershipCheck and main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001557 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001558 main.log.warn( "Mastership of switch %s changed" % switchDPID )
Jon Hall8f89dda2015-01-22 16:03:33 -08001559 mastershipCheck = main.FALSE
1560 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001561 main.log.report( "Mastership of Switches was not changed" )
1562 utilities.assert_equals(
1563 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001564 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001565 onpass="Mastership of Switches was not changed",
1566 onfail="Mastership of some switches changed" )
1567 # NOTE: we expect mastership to change on controller failure
Jon Hall8f89dda2015-01-22 16:03:33 -08001568 mastershipCheck = mastershipCheck and consistentMastership
Jon Hall73cf9cc2014-11-20 22:28:38 -08001569
Jon Hall6aec96b2015-01-19 14:49:31 -08001570 main.step( "Get the intents and compare across all nodes" )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001571 ONOSIntents = []
Jon Hall8f89dda2015-01-22 16:03:33 -08001572 intentCheck = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -07001573 consistentIntents = True
1574 intentsResults = True
1575 threads = []
1576 for i in range( numControllers ):
1577 t = main.Thread( target=CLIs[i].intents,
1578 name="intents-" + str( i ),
1579 args=[],
1580 kwargs={ 'jsonFormat': True } )
1581 threads.append( t )
1582 t.start()
1583
1584 for t in threads:
1585 t.join()
1586 ONOSIntents.append( t.result )
1587
1588 for i in range( numControllers ):
1589 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1590 main.log.report( "Error in getting ONOS" + str( i + 1 ) +
1591 " intents" )
1592 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1593 repr( ONOSIntents[ i ] ) )
1594 intentsResults = False
Jon Hall6aec96b2015-01-19 14:49:31 -08001595 utilities.assert_equals(
Jon Hall5cfd23c2015-03-19 11:40:57 -07001596 expect=True,
1597 actual=intentsResults,
1598 onpass="No error in reading intents output",
1599 onfail="Error in reading intents from ONOS" )
1600
1601 main.step( "Check for consistency in Intents from each controller" )
1602 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1603 main.log.report( "Intents are consistent across all ONOS " +
1604 "nodes" )
1605 else:
1606 consistentIntents = False
1607 utilities.assert_equals(
1608 expect=True,
1609 actual=consistentIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001610 onpass="Intents are consistent across all ONOS nodes",
1611 onfail="ONOS nodes have different views of intents" )
Jon Hall1b8f54a2015-02-04 13:24:20 -08001612 intentStates = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001613 for node in ONOSIntents: # Iter through ONOS nodes
Jon Hall1b8f54a2015-02-04 13:24:20 -08001614 nodeStates = []
Jon Hall58c76b72015-02-23 11:09:24 -08001615 # Iter through intents of a node
Jon Hall5cfd23c2015-03-19 11:40:57 -07001616 try:
1617 for intent in json.loads( node ):
1618 nodeStates.append( intent[ 'state' ] )
1619 except ( ValueError, TypeError ):
1620 main.log.exception( "Error in parsing intents" )
1621 main.log.error( repr( node ) )
Jon Hall1b8f54a2015-02-04 13:24:20 -08001622 intentStates.append( nodeStates )
1623 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1624 main.log.info( dict( out ) )
1625
Jon Hall5cfd23c2015-03-19 11:40:57 -07001626 if intentsResults and not consistentIntents:
1627 for i in range( numControllers ):
1628 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1629 main.log.warn( json.dumps(
1630 json.loads( ONOSIntents[ i ] ),
1631 sort_keys=True,
1632 indent=4,
1633 separators=( ',', ': ' ) ) )
1634 elif intentsResults and consistentIntents:
1635 intentCheck = main.TRUE
1636
Jon Hall58c76b72015-02-23 11:09:24 -08001637 # NOTE: Store has no durability, so intents are lost across system
1638 # restarts
Jon Hall6aec96b2015-01-19 14:49:31 -08001639 """
1640 main.step( "Compare current intents with intents before the failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001641 # NOTE: this requires case 5 to pass for intentState to be set.
Jon Hall94fd0472014-12-08 11:52:42 -08001642 # maybe we should stop the test if that fails?
Jon Hall1b8f54a2015-02-04 13:24:20 -08001643 sameIntents = main.TRUE
Jon Hall5cfd23c2015-03-19 11:40:57 -07001644 if intentState and intentState == ONOSIntents[ 0 ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001645 sameIntents = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001646 main.log.report( "Intents are consistent with before failure" )
1647 # TODO: possibly the states have changed? we may need to figure out
Jon Hall5cfd23c2015-03-19 11:40:57 -07001648 # what the acceptable states are
Jon Hall73cf9cc2014-11-20 22:28:38 -08001649 else:
Jon Hall669173b2014-12-17 11:36:30 -08001650 try:
Jon Hall5cfd23c2015-03-19 11:40:57 -07001651 main.log.warn( "ONOS intents: " )
1652 main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1653 sort_keys=True, indent=4,
1654 separators=( ',', ': ' ) ) )
1655 except ( ValueError, TypeError ):
1656 main.log.exception( "Exception printing intents" )
1657 main.log.warn( repr( ONOSIntents[0] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001658 sameIntents = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001659 utilities.assert_equals(
1660 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001661 actual=sameIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001662 onpass="Intents are consistent with before failure",
1663 onfail="The Intents changed during failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001664 intentCheck = intentCheck and sameIntents
Jon Hall6aec96b2015-01-19 14:49:31 -08001665 """
1666 main.step( "Get the OF Table entries and compare to before " +
1667 "component failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001668 FlowTables = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001669 flows2 = []
1670 for i in range( 28 ):
1671 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001672 tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
1673 flows2.append( tmpFlows )
1674 tempResult = main.Mininet2.flowComp(
Jon Hall6aec96b2015-01-19 14:49:31 -08001675 flow1=flows[ i ],
Jon Hall8f89dda2015-01-22 16:03:33 -08001676 flow2=tmpFlows )
1677 FlowTables = FlowTables and tempResult
1678 if FlowTables == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001679 main.log.info( "Differences in flow table for switch: s" +
1680 str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001681 if FlowTables == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001682 main.log.report( "No changes were found in the flow tables" )
1683 utilities.assert_equals(
1684 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001685 actual=FlowTables,
Jon Hall6aec96b2015-01-19 14:49:31 -08001686 onpass="No changes were found in the flow tables",
1687 onfail="Changes were found in the flow tables" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001688
Jon Hall6aec96b2015-01-19 14:49:31 -08001689 main.step( "Check the continuous pings to ensure that no packets " +
1690 "were dropped during component failure" )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001691 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1692 main.params[ 'TESTONIP' ] )
Jon Hall8f89dda2015-01-22 16:03:33 -08001693 LossInPings = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001694 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1695 for i in range( 8, 18 ):
1696 main.log.info(
1697 "Checking for a loss in pings along flow from s" +
1698 str( i ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001699 LossInPings = main.Mininet2.checkForLoss(
Jon Hall6aec96b2015-01-19 14:49:31 -08001700 "/tmp/ping.h" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001701 str( i ) ) or LossInPings
1702 if LossInPings == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001703 main.log.info( "Loss in ping detected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001704 elif LossInPings == main.ERROR:
Jon Hall6aec96b2015-01-19 14:49:31 -08001705 main.log.info( "There are multiple mininet process running" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001706 elif LossInPings == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001707 main.log.info( "No Loss in the pings" )
1708 main.log.report( "No loss of dataplane connectivity" )
1709 utilities.assert_equals(
1710 expect=main.FALSE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001711 actual=LossInPings,
Jon Hall6aec96b2015-01-19 14:49:31 -08001712 onpass="No Loss of connectivity",
1713 onfail="Loss of dataplane connectivity detected" )
Jon Hall58c76b72015-02-23 11:09:24 -08001714 # NOTE: Since intents are not persisted with IntnentStore,
1715 # we expect loss in dataplane connectivity
Jon Hall8f89dda2015-01-22 16:03:33 -08001716 LossInPings = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001717
Jon Hall6aec96b2015-01-19 14:49:31 -08001718 # Test of LeadershipElection
Jon Hall8f89dda2015-01-22 16:03:33 -08001719 leaderList = []
1720 leaderResult = main.TRUE
Jon Hall5cfd23c2015-03-19 11:40:57 -07001721 for cli in CLIs:
1722 leaderN = cli.electionTestLeader()
Jon Hall8f89dda2015-01-22 16:03:33 -08001723 leaderList.append( leaderN )
Jon Hall669173b2014-12-17 11:36:30 -08001724 if leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001725 # error in response
1726 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001727 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001728 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001729 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001730 elif leaderN is None:
Jon Hall5cfd23c2015-03-19 11:40:57 -07001731 main.log.report( cli.name +
Jon Hall6aec96b2015-01-19 14:49:31 -08001732 " shows no leader for the election-app was" +
1733 " elected after the old one died" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001734 leaderResult = main.FALSE
1735 if len( set( leaderList ) ) != 1:
1736 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001737 main.log.error(
1738 "Inconsistent view of leader for the election test app" )
1739 # TODO: print the list
Jon Hall8f89dda2015-01-22 16:03:33 -08001740 if leaderResult:
1741 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001742 "view of leader across listeners and a new " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001743 "leader was re-elected if applicable )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001744 utilities.assert_equals(
1745 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001746 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001747 onpass="Leadership election passed",
1748 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08001749
Jon Hall8f89dda2015-01-22 16:03:33 -08001750 result = ( mastershipCheck and intentCheck and FlowTables and
1751 ( not LossInPings ) and rolesNotNull and leaderResult )
Jon Hall6aec96b2015-01-19 14:49:31 -08001752 result = int( result )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001753 if result == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001754 main.log.report( "Constant State Tests Passed" )
1755 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08001756 onpass="Constant State Tests Passed",
1757 onfail="Constant state tests failed" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001758
Jon Hall6aec96b2015-01-19 14:49:31 -08001759 def CASE8( self, main ):
1760 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001761 Compare topo
Jon Hall6aec96b2015-01-19 14:49:31 -08001762 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001763 import sys
Jon Hall6aec96b2015-01-19 14:49:31 -08001764 # FIXME add this path to params
1765 sys.path.append( "/home/admin/sts" )
1766 # assumes that sts is already in you PYTHONPATH
1767 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -08001768 import json
1769 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07001770 assert numControllers, "numControllers not defined"
1771 assert main, "main not defined"
1772 assert utilities.assert_equals, "utilities.assert_equals not defined"
1773 assert CLIs, "CLIs not defined"
1774 assert nodes, "nodes not defined"
Jon Hall73cf9cc2014-11-20 22:28:38 -08001775
Jon Hall6aec96b2015-01-19 14:49:31 -08001776 description = "Compare ONOS Topology view to Mininet topology"
1777 main.case( description )
1778 main.log.report( description )
1779 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001780 ctrls = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001781 for node in nodes:
1782 temp = ( node, node.name, node.ip_address, 6633 )
1783 ctrls.append( temp )
1784 MNTopo = TestONTopology( main.Mininet1, ctrls )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001785
Jon Hall6aec96b2015-01-19 14:49:31 -08001786 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001787 devicesResults = main.TRUE
1788 portsResults = main.TRUE
1789 linksResults = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -08001790 hostsResults = main.TRUE
Jon Hall8f89dda2015-01-22 16:03:33 -08001791 topoResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001792 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -08001793 count = 0
Jon Hall6aec96b2015-01-19 14:49:31 -08001794 main.step( "Collecting topology information from ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001795 startTime = time.time()
Jon Hall21270ac2015-02-16 17:59:55 -08001796 # Give time for Gossip to work
Jon Hall8f89dda2015-01-22 16:03:33 -08001797 while topoResult == main.FALSE and elapsed < 60:
Jon Hall5cfd23c2015-03-19 11:40:57 -07001798 count += 1
Jon Hall94fd0472014-12-08 11:52:42 -08001799 if count > 1:
Jon Hall5cfd23c2015-03-19 11:40:57 -07001800 # TODO: Deprecate STS usage
Jon Hall58c76b72015-02-23 11:09:24 -08001801 MNTopo = TestONTopology( main.Mininet1, ctrls )
Jon Hall8f89dda2015-01-22 16:03:33 -08001802 cliStart = time.time()
Jon Hall94fd0472014-12-08 11:52:42 -08001803 devices = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001804 threads = []
1805 for i in range( numControllers ):
1806 t = main.Thread( target=CLIs[i].devices,
1807 name="devices-" + str( i ),
1808 args=[ ] )
1809 threads.append( t )
1810 t.start()
1811
1812 for t in threads:
1813 t.join()
1814 devices.append( t.result )
Jon Hall94fd0472014-12-08 11:52:42 -08001815 hosts = []
Jon Hall58c76b72015-02-23 11:09:24 -08001816 ipResult = main.TRUE
Jon Hall5cfd23c2015-03-19 11:40:57 -07001817 threads = []
1818 for i in range( numControllers ):
1819 t = main.Thread( target=CLIs[i].hosts,
1820 name="hosts-" + str( i ),
1821 args=[ ] )
1822 threads.append( t )
1823 t.start()
1824
1825 for t in threads:
1826 t.join()
1827 try:
1828 hosts.append( json.loads( t.result ) )
1829 except ( ValueError, TypeError ):
1830 main.log.exception( "Error parsing hosts results" )
1831 main.log.error( repr( t.result ) )
Jon Hall529a37f2015-01-28 10:02:00 -08001832 for controller in range( 0, len( hosts ) ):
1833 controllerStr = str( controller + 1 )
1834 for host in hosts[ controller ]:
Jon Hall58c76b72015-02-23 11:09:24 -08001835 if host is None or host.get( 'ips', [] ) == []:
Jon Hall529a37f2015-01-28 10:02:00 -08001836 main.log.error(
1837 "DEBUG:Error with host ips on controller" +
1838 controllerStr + ": " + str( host ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001839 ipResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001840 ports = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001841 threads = []
1842 for i in range( numControllers ):
1843 t = main.Thread( target=CLIs[i].ports,
1844 name="ports-" + str( i ),
1845 args=[ ] )
1846 threads.append( t )
1847 t.start()
1848
1849 for t in threads:
1850 t.join()
1851 ports.append( t.result )
Jon Hall94fd0472014-12-08 11:52:42 -08001852 links = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001853 threads = []
1854 for i in range( numControllers ):
1855 t = main.Thread( target=CLIs[i].links,
1856 name="links-" + str( i ),
1857 args=[ ] )
1858 threads.append( t )
1859 t.start()
1860
1861 for t in threads:
1862 t.join()
1863 links.append( t.result )
Jon Hall94fd0472014-12-08 11:52:42 -08001864 clusters = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001865 threads = []
1866 for i in range( numControllers ):
1867 t = main.Thread( target=CLIs[i].clusters,
1868 name="clusters-" + str( i ),
1869 args=[ ] )
1870 threads.append( t )
1871 t.start()
1872
1873 for t in threads:
1874 t.join()
1875 clusters.append( t.result )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001876
Jon Hall8f89dda2015-01-22 16:03:33 -08001877 elapsed = time.time() - startTime
1878 cliTime = time.time() - cliStart
1879 print "CLI time: " + str( cliTime )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001880
Jon Hall8f89dda2015-01-22 16:03:33 -08001881 for controller in range( numControllers ):
1882 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001883 if devices[ controller ] or "Error" not in devices[
1884 controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001885 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001886 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001887 json.loads( devices[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001888 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001889 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001890 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001891 actual=currentDevicesResult,
1892 onpass="ONOS" + controllerStr +
1893 " Switches view is correct",
1894 onfail="ONOS" + controllerStr +
1895 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001896
Jon Hall6aec96b2015-01-19 14:49:31 -08001897 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001898 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001899 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001900 json.loads( ports[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001901 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001902 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001903 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001904 actual=currentPortsResult,
1905 onpass="ONOS" + controllerStr +
1906 " ports view is correct",
1907 onfail="ONOS" + controllerStr +
1908 " ports view is incorrect" )
Jon Hall94fd0472014-12-08 11:52:42 -08001909
Jon Hall6aec96b2015-01-19 14:49:31 -08001910 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001911 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001912 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001913 json.loads( links[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001914 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001915 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001916 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001917 actual=currentLinksResult,
1918 onpass="ONOS" + controllerStr +
1919 " links view is correct",
1920 onfail="ONOS" + controllerStr +
1921 " links view is incorrect" )
1922
1923 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1924 currentHostsResult = main.Mininet1.compareHosts(
1925 MNTopo, hosts[ controller ] )
1926 else:
1927 currentHostsResult = main.FALSE
1928 utilities.assert_equals( expect=main.TRUE,
1929 actual=currentHostsResult,
1930 onpass="ONOS" + controllerStr +
1931 " hosts exist in Mininet",
1932 onfail="ONOS" + controllerStr +
1933 " hosts don't match Mininet" )
1934
1935 devicesResults = devicesResults and currentDevicesResult
1936 portsResults = portsResults and currentPortsResult
1937 linksResults = linksResults and currentLinksResult
1938 hostsResults = hostsResults and currentHostsResult
Jon Hall94fd0472014-12-08 11:52:42 -08001939
Jon Hall529a37f2015-01-28 10:02:00 -08001940 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001941
Jon Hall6aec96b2015-01-19 14:49:31 -08001942 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -08001943 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001944 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001945 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001946 if "Error" not in hosts[ controller ]:
1947 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001948 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001949 else: # hosts not consistent
Jon Hall8f89dda2015-01-22 16:03:33 -08001950 main.log.report( "hosts from ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001951 " is inconsistent with ONOS1" )
1952 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001953 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001954
1955 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001956 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001957 controllerStr )
1958 consistentHostsResult = main.FALSE
1959 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001960 " hosts response: " +
1961 repr( hosts[ controller ] ) )
1962 utilities.assert_equals(
1963 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001964 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001965 onpass="Hosts view is consistent across all ONOS nodes",
1966 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001967
Jon Hall6aec96b2015-01-19 14:49:31 -08001968 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001969 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001970 for controller in range( len( clusters ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001971 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001972 if "Error" not in clusters[ controller ]:
1973 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001974 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001975 else: # clusters not consistent
1976 main.log.report( "clusters from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001977 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001978 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001979 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001980
1981 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001982 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001983 "from ONOS" + controllerStr )
1984 consistentClustersResult = main.FALSE
1985 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001986 " clusters response: " +
1987 repr( clusters[ controller ] ) )
1988 utilities.assert_equals(
1989 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001990 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001991 onpass="Clusters view is consistent across all ONOS nodes",
1992 onfail="ONOS nodes have different views of clusters" )
1993 # there should always only be one cluster
Jon Hall5cfd23c2015-03-19 11:40:57 -07001994 try:
1995 numClusters = len( json.loads( clusters[ 0 ] ) )
1996 except ( ValueError, TypeError ):
1997 main.log.exception( "Error parsing clusters[0]: " +
1998 repr( clusters[0] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001999 clusterResults = main.FALSE
2000 if numClusters == 1:
2001 clusterResults = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002002 utilities.assert_equals(
2003 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08002004 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08002005 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08002006 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08002007
Jon Hall8f89dda2015-01-22 16:03:33 -08002008 topoResult = ( devicesResults and portsResults and linksResults
Jon Hall58c76b72015-02-23 11:09:24 -08002009 and hostsResults and consistentHostsResult
2010 and consistentClustersResult and clusterResults
2011 and ipResult )
Jon Hall94fd0472014-12-08 11:52:42 -08002012
Jon Hall8f89dda2015-01-22 16:03:33 -08002013 topoResult = topoResult and int( count <= 2 )
2014 note = "note it takes about " + str( int( cliTime ) ) + \
2015 " seconds for the test to make all the cli calls to fetch " +\
2016 "the topology from each ONOS instance"
Jon Hall1b8f54a2015-02-04 13:24:20 -08002017 main.log.info(
Jon Hall8f89dda2015-01-22 16:03:33 -08002018 "Very crass estimate for topology discovery/convergence( " +
2019 str( note ) + " ): " + str( elapsed ) + " seconds, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002020 str( count ) + " tries" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002021 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08002022 onpass="Topology Check Test successful",
2023 onfail="Topology Check Test NOT successful" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002024 if topoResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002025 main.log.report( "ONOS topology view matches Mininet topology" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002026
Jon Hall6aec96b2015-01-19 14:49:31 -08002027 def CASE9( self, main ):
2028 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002029 Link s3-s28 down
Jon Hall6aec96b2015-01-19 14:49:31 -08002030 """
2031 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002032 assert numControllers, "numControllers not defined"
2033 assert main, "main not defined"
2034 assert utilities.assert_equals, "utilities.assert_equals not defined"
2035 assert CLIs, "CLIs not defined"
2036 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08002037 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08002038
Jon Hall8f89dda2015-01-22 16:03:33 -08002039 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002040
Jon Hall6aec96b2015-01-19 14:49:31 -08002041 description = "Turn off a link to ensure that Link Discovery " +\
Jon Hall58c76b72015-02-23 11:09:24 -08002042 "is working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002043 main.log.report( description )
2044 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002045
Jon Hall6aec96b2015-01-19 14:49:31 -08002046 main.step( "Kill Link between s3 and s28" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002047 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
Jon Hall58c76b72015-02-23 11:09:24 -08002048 main.log.info( "Waiting " + str( linkSleep ) +
2049 " seconds for link down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002050 time.sleep( linkSleep )
2051 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002052 onpass="Link down successful",
Jon Hall58c76b72015-02-23 11:09:24 -08002053 onfail="Failed to bring link down" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002054 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -08002055
Jon Hall6aec96b2015-01-19 14:49:31 -08002056 def CASE10( self, main ):
2057 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002058 Link s3-s28 up
Jon Hall6aec96b2015-01-19 14:49:31 -08002059 """
2060 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002061 assert numControllers, "numControllers not defined"
2062 assert main, "main not defined"
2063 assert utilities.assert_equals, "utilities.assert_equals not defined"
2064 assert CLIs, "CLIs not defined"
2065 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08002066 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08002067
Jon Hall8f89dda2015-01-22 16:03:33 -08002068 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002069
Jon Hall6aec96b2015-01-19 14:49:31 -08002070 description = "Restore a link to ensure that Link Discovery is " + \
Jon Hall63604932015-02-26 17:09:50 -08002071 "working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002072 main.log.report( description )
2073 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002074
Jon Hall6aec96b2015-01-19 14:49:31 -08002075 main.step( "Bring link between s3 and s28 back up" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002076 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
Jon Hall58c76b72015-02-23 11:09:24 -08002077 main.log.info( "Waiting " + str( linkSleep ) +
2078 " seconds for link up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002079 time.sleep( linkSleep )
2080 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002081 onpass="Link up successful",
Jon Hall58c76b72015-02-23 11:09:24 -08002082 onfail="Failed to bring link up" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002083 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -08002084
Jon Hall6aec96b2015-01-19 14:49:31 -08002085 def CASE11( self, main ):
2086 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002087 Switch Down
Jon Hall6aec96b2015-01-19 14:49:31 -08002088 """
2089 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08002090 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002091 assert numControllers, "numControllers not defined"
2092 assert main, "main not defined"
2093 assert utilities.assert_equals, "utilities.assert_equals not defined"
2094 assert CLIs, "CLIs not defined"
2095 assert nodes, "nodes not defined"
Jon Hall73cf9cc2014-11-20 22:28:38 -08002096
Jon Hall8f89dda2015-01-22 16:03:33 -08002097 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002098
2099 description = "Killing a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002100 main.log.report( description )
2101 main.case( description )
2102 switch = main.params[ 'kill' ][ 'switch' ]
2103 switchDPID = main.params[ 'kill' ][ 'dpid' ]
Jon Hall73cf9cc2014-11-20 22:28:38 -08002104
Jon Hall6aec96b2015-01-19 14:49:31 -08002105 # TODO: Make this switch parameterizable
2106 main.step( "Kill " + switch )
2107 main.log.report( "Deleting " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08002108 main.Mininet1.delSwitch( switch )
2109 main.log.info( "Waiting " + str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002110 " seconds for switch down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002111 time.sleep( switchSleep )
2112 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002113 # Peek at the deleted switch
2114 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08002115 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002116 if device and device[ 'available' ] is False:
Jon Hall94fd0472014-12-08 11:52:42 -08002117 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002118 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002119 onpass="Kill switch successful",
Jon Hall58c76b72015-02-23 11:09:24 -08002120 onfail="Failed to kill switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002121
Jon Hall6aec96b2015-01-19 14:49:31 -08002122 def CASE12( self, main ):
2123 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002124 Switch Up
Jon Hall6aec96b2015-01-19 14:49:31 -08002125 """
2126 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -08002127 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002128 assert numControllers, "numControllers not defined"
2129 assert main, "main not defined"
2130 assert utilities.assert_equals, "utilities.assert_equals not defined"
2131 assert CLIs, "CLIs not defined"
2132 assert nodes, "nodes not defined"
2133 assert ONOS1Port, "ONOS1Port not defined"
2134 assert ONOS2Port, "ONOS2Port not defined"
2135 assert ONOS3Port, "ONOS3Port not defined"
2136 assert ONOS4Port, "ONOS4Port not defined"
2137 assert ONOS5Port, "ONOS5Port not defined"
2138 assert ONOS6Port, "ONOS6Port not defined"
2139 assert ONOS7Port, "ONOS7Port not defined"
Jon Hall669173b2014-12-17 11:36:30 -08002140
Jon Hall8f89dda2015-01-22 16:03:33 -08002141 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall6aec96b2015-01-19 14:49:31 -08002142 switch = main.params[ 'kill' ][ 'switch' ]
2143 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2144 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hall73cf9cc2014-11-20 22:28:38 -08002145 description = "Adding a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002146 main.log.report( description )
2147 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002148
Jon Hall6aec96b2015-01-19 14:49:31 -08002149 main.step( "Add back " + switch )
2150 main.log.report( "Adding back " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08002151 main.Mininet1.addSwitch( switch, dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002152 for peer in links:
Jon Hall8f89dda2015-01-22 16:03:33 -08002153 main.Mininet1.addLink( switch, peer )
Jon Hall58c76b72015-02-23 11:09:24 -08002154 main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
2155 count=numControllers,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002156 ip1=nodes[ 0 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002157 port1=ONOS1Port,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002158 ip2=nodes[ 1 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002159 port2=ONOS2Port,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002160 ip3=nodes[ 2 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002161 port3=ONOS3Port,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002162 ip4=nodes[ 3 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002163 port4=ONOS4Port,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002164 ip5=nodes[ 4 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002165 port5=ONOS5Port,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002166 ip6=nodes[ 5 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002167 port6=ONOS6Port,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002168 ip7=nodes[ 6 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002169 port7=ONOS7Port )
2170 main.log.info( "Waiting " + str( switchSleep ) +
2171 " seconds for switch up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002172 time.sleep( switchSleep )
2173 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002174 # Peek at the deleted switch
2175 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08002176 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002177 if device and device[ 'available' ]:
Jon Hall94fd0472014-12-08 11:52:42 -08002178 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002179 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall5cfd23c2015-03-19 11:40:57 -07002180 onpass="add switch successful",
Jon Hall58c76b72015-02-23 11:09:24 -08002181 onfail="Failed to add switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002182
Jon Hall6aec96b2015-01-19 14:49:31 -08002183 def CASE13( self, main ):
2184 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002185 Clean up
Jon Hall6aec96b2015-01-19 14:49:31 -08002186 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002187 import os
2188 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002189 assert numControllers, "numControllers not defined"
2190 assert main, "main not defined"
2191 assert utilities.assert_equals, "utilities.assert_equals not defined"
2192 assert CLIs, "CLIs not defined"
2193 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08002194
2195 # printing colors to terminal
Jon Hall5cfd23c2015-03-19 11:40:57 -07002196 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2197 'blue': '\033[94m', 'green': '\033[92m',
2198 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
Jon Hall73cf9cc2014-11-20 22:28:38 -08002199 description = "Test Cleanup"
Jon Hall6aec96b2015-01-19 14:49:31 -08002200 main.log.report( description )
2201 main.case( description )
2202 main.step( "Killing tcpdumps" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002203 main.Mininet2.stopTcpdump()
Jon Hall73cf9cc2014-11-20 22:28:38 -08002204
Jon Hall6aec96b2015-01-19 14:49:31 -08002205 main.step( "Copying MN pcap and ONOS log files to test station" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002206 testname = main.TEST
Jon Hall8f89dda2015-01-22 16:03:33 -08002207 teststationUser = main.params[ 'TESTONUSER' ]
2208 teststationIP = main.params[ 'TESTONIP' ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002209 # NOTE: MN Pcap file is being saved to ~/packet_captures
Jon Hall73cf9cc2014-11-20 22:28:38 -08002210 # scp this file as MN and TestON aren't necessarily the same vm
Jon Hall6aec96b2015-01-19 14:49:31 -08002211 # FIXME: scp
2212 # mn files
2213 # TODO: Load these from params
2214 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002215 logFolder = "/opt/onos/log/"
2216 logFiles = [ "karaf.log", "karaf.log.1" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002217 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002218 dstDir = "~/packet_captures/"
2219 for f in logFiles:
Jon Hall5cfd23c2015-03-19 11:40:57 -07002220 for node in nodes:
2221 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2222 ":" + logFolder + f + " " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002223 teststationUser + "@" +
2224 teststationIP + ":" +
2225 dstDir + str( testname ) +
Jon Hall5cfd23c2015-03-19 11:40:57 -07002226 "-" + node.name + "-" + f )
Jon Hall58c76b72015-02-23 11:09:24 -08002227 main.ONOSbench.handle.expect( "\$" )
2228
Jon Hall6aec96b2015-01-19 14:49:31 -08002229 # std*.log's
2230 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002231 logFolder = "/opt/onos/var/"
2232 logFiles = [ "stderr.log", "stdout.log" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002233 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002234 dstDir = "~/packet_captures/"
2235 for f in logFiles:
Jon Hall5cfd23c2015-03-19 11:40:57 -07002236 for node in nodes:
2237 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2238 ":" + logFolder + f + " " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002239 teststationUser + "@" +
2240 teststationIP + ":" +
2241 dstDir + str( testname ) +
Jon Hall5cfd23c2015-03-19 11:40:57 -07002242 "-" + node.name + "-" + f )
Jon Hall58c76b72015-02-23 11:09:24 -08002243 main.ONOSbench.handle.expect( "\$" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002244 # sleep so scp can finish
2245 time.sleep( 10 )
Jon Hall5cfd23c2015-03-19 11:40:57 -07002246
2247 main.step( "Stopping Mininet" )
Jon Hall58c76b72015-02-23 11:09:24 -08002248 main.Mininet1.stopNet()
Jon Hall5cfd23c2015-03-19 11:40:57 -07002249
2250 main.step( "Checking ONOS Logs for errors" )
2251 for node in nodes:
2252 print colors[ 'purple' ] + "Checking logs for errors on " + \
2253 node.name + ":" + colors[ 'end' ]
2254 print main.ONOSbench.checkLogs( node.ip_address )
2255
Jon Hall6aec96b2015-01-19 14:49:31 -08002256 main.step( "Packing and rotating pcap archives" )
2257 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002258
Jon Hall6aec96b2015-01-19 14:49:31 -08002259 # TODO: actually check something here
2260 utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08002261 onpass="Test cleanup successful",
2262 onfail="Test cleanup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002263
Jon Hall6aec96b2015-01-19 14:49:31 -08002264 def CASE14( self, main ):
2265 """
Jon Hall669173b2014-12-17 11:36:30 -08002266 start election app on all onos nodes
Jon Hall6aec96b2015-01-19 14:49:31 -08002267 """
Jon Hall5cfd23c2015-03-19 11:40:57 -07002268 assert numControllers, "numControllers not defined"
2269 assert main, "main not defined"
2270 assert utilities.assert_equals, "utilities.assert_equals not defined"
2271 assert CLIs, "CLIs not defined"
2272 assert nodes, "nodes not defined"
2273
Jon Hall8f89dda2015-01-22 16:03:33 -08002274 leaderResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002275 # install app on onos 1
2276 main.log.info( "Install leadership election app" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002277 main.ONOScli1.featureInstall( "onos-app-election" )
Jon Hall5cfd23c2015-03-19 11:40:57 -07002278 leader = nodes[0].ip_address
Jon Hall6aec96b2015-01-19 14:49:31 -08002279 # wait for election
2280 # check for leader
Jon Hall5cfd23c2015-03-19 11:40:57 -07002281 leader1 = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002282 # verify leader is ONOS1
Jon Hall5cfd23c2015-03-19 11:40:57 -07002283 if leader1 == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08002284 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08002285 pass
Jon Hall5cfd23c2015-03-19 11:40:57 -07002286 elif leader1 is None:
Jon Hall6aec96b2015-01-19 14:49:31 -08002287 # No leader elected
2288 main.log.report( "No leader was elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002289 leaderResult = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -07002290 elif leader1 == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002291 # error in response
2292 # TODO: add check for "Command not found:" in the driver, this
2293 # means the app isn't loaded
Jon Hall8f89dda2015-01-22 16:03:33 -08002294 main.log.report( "Something is wrong with electionTestLeader" +
Jon Hall6aec96b2015-01-19 14:49:31 -08002295 " function, check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002296 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002297 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002298 # error in response
2299 main.log.report(
Jon Hall8f89dda2015-01-22 16:03:33 -08002300 "Unexpected response from electionTestLeader function:'" +
Jon Hall5cfd23c2015-03-19 11:40:57 -07002301 str( leader1 ) + "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002302 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002303
Jon Hall6aec96b2015-01-19 14:49:31 -08002304 # install on other nodes and check for leader.
Jon Hall5cfd23c2015-03-19 11:40:57 -07002305 # Leader should be ONOS1 and each app should show the same leader
2306 for cli in CLIs[ 1: ]:
2307 cli.featureInstall( "onos-app-election" )
2308 leaderN = cli.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002309 # verify leader is ONOS1
Jon Hall5cfd23c2015-03-19 11:40:57 -07002310 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08002311 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08002312 pass
2313 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002314 # error in response
2315 # TODO: add check for "Command not found:" in the driver, this
2316 # means the app isn't loaded
2317 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002318 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08002319 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002320 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002321 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08002322 leaderResult = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -07002323 main.log.report( cli.name + " sees " + str( leaderN ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002324 " as the leader of the election app. Leader" +
2325 " should be " +
2326 str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002327 if leaderResult:
Jon Hall6aec96b2015-01-19 14:49:31 -08002328 main.log.report( "Leadership election tests passed( consistent " +
2329 "view of leader across listeners and a leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002330 "was elected )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002331 utilities.assert_equals(
2332 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002333 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002334 onpass="Leadership election passed",
2335 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002336
Jon Hall6aec96b2015-01-19 14:49:31 -08002337 def CASE15( self, main ):
2338 """
Jon Hall669173b2014-12-17 11:36:30 -08002339 Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -08002340 """
Jon Hall5cfd23c2015-03-19 11:40:57 -07002341 assert numControllers, "numControllers not defined"
2342 assert main, "main not defined"
2343 assert utilities.assert_equals, "utilities.assert_equals not defined"
2344 assert CLIs, "CLIs not defined"
2345 assert nodes, "nodes not defined"
2346
Jon Hall8f89dda2015-01-22 16:03:33 -08002347 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002348 description = "Check that Leadership Election is still functional"
Jon Hall6aec96b2015-01-19 14:49:31 -08002349 main.log.report( description )
2350 main.case( description )
2351 main.step( "Find current leader and withdraw" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002352 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002353 # TODO: do some sanity checking on leader before using it
Jon Hall8f89dda2015-01-22 16:03:33 -08002354 withdrawResult = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -07002355 if leader is None or leader == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002356 main.log.report(
2357 "Leader for the election app should be an ONOS node," +
Jon Hall58c76b72015-02-23 11:09:24 -08002358 "instead got '" + str( leader ) + "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002359 leaderResult = main.FALSE
Jon Hall63604932015-02-26 17:09:50 -08002360 oldLeader = None
Jon Hall5cfd23c2015-03-19 11:40:57 -07002361 for i in range( len( CLIs ) ):
2362 if leader == nodes[ i ].ip_address:
2363 oldLeader = CLIs[ i ]
2364 break
Jon Hall63604932015-02-26 17:09:50 -08002365 else:
Jon Hall5cfd23c2015-03-19 11:40:57 -07002366 main.log.error( "Leader election, could not find current leader" )
Jon Hall63604932015-02-26 17:09:50 -08002367 if oldLeader:
2368 withdrawResult = oldLeader.electionTestWithdraw()
Jon Hall6aec96b2015-01-19 14:49:31 -08002369 utilities.assert_equals(
2370 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002371 actual=withdrawResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002372 onpass="App was withdrawn from election",
2373 onfail="App was not withdrawn from election" )
Jon Hall669173b2014-12-17 11:36:30 -08002374
Jon Hall6aec96b2015-01-19 14:49:31 -08002375 main.step( "Make sure new leader is elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002376 leaderList = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07002377 for cli in CLIs:
2378 leaderN = cli.electionTestLeader()
2379 leaderList.append( leaderN )
Jon Hall669173b2014-12-17 11:36:30 -08002380 if leaderN == leader:
Jon Hall5cfd23c2015-03-19 11:40:57 -07002381 main.log.report( cli.name + " still sees " + str( leader ) +
2382 " as leader after they withdrew" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002383 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002384 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002385 # error in response
2386 # TODO: add check for "Command not found:" in the driver, this
Jon Hall5cfd23c2015-03-19 11:40:57 -07002387 # means the app isn't loaded
Jon Hall6aec96b2015-01-19 14:49:31 -08002388 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002389 "electionTestLeader function, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002390 "check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002391 leaderResult = main.FALSE
2392 consistentLeader = main.FALSE
2393 if len( set( leaderList ) ) == 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08002394 main.log.info( "Each Election-app sees '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08002395 str( leaderList[ 0 ] ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002396 "' as the leader" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002397 consistentLeader = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002398 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002399 main.log.report(
2400 "Inconsistent responses for leader of Election-app:" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002401 for n in range( len( leaderList ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002402 main.log.report( "ONOS" + str( n + 1 ) + " response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002403 str( leaderList[ n ] ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -07002404 leaderResult = leaderResult and consistentLeader
Jon Hall8f89dda2015-01-22 16:03:33 -08002405 if leaderResult:
2406 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002407 "view of leader across listeners and a new " +
2408 "leader was elected when the old leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002409 "resigned )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002410 utilities.assert_equals(
2411 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002412 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002413 onpass="Leadership election passed",
2414 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002415
Jon Hall58c76b72015-02-23 11:09:24 -08002416 main.step( "Run for election on old leader( just so everyone " +
2417 "is in the hat )" )
Jon Hall63604932015-02-26 17:09:50 -08002418 if oldLeader:
2419 runResult = oldLeader.electionTestRun()
2420 else:
2421 runResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002422 utilities.assert_equals(
2423 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002424 actual=runResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002425 onpass="App re-ran for election",
2426 onfail="App failed to run for election" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002427 if consistentLeader == main.TRUE:
2428 afterRun = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002429 # verify leader didn't just change
Jon Hall8f89dda2015-01-22 16:03:33 -08002430 if afterRun == leaderList[ 0 ]:
2431 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002432 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08002433 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002434 # TODO: assert on run and withdraw results?
Jon Hall669173b2014-12-17 11:36:30 -08002435
Jon Hall6aec96b2015-01-19 14:49:31 -08002436 utilities.assert_equals(
2437 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002438 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002439 onpass="Leadership election passed",
2440 onfail="Something went wrong with Leadership election after " +
2441 "the old leader re-ran for election" )