blob: 584232a0df817cda53a60a140ad5de874bf2b588 [file] [log] [blame]
Jon Hall85794ff2015-07-08 14:12:30 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2015 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070021"""
Jon Hall85794ff2015-07-08 14:12:30 -070022Description: This test is to determine if a single
23 instance ONOS 'cluster' can handle a restart
24
25List of test cases:
26CASE1: Compile ONOS and push it to the test machines
27CASE2: Assign devices to controllers
28CASE21: Assign mastership to controllers
29CASE3: Assign intents
30CASE4: Ping across added host intents
31CASE5: Reading state of ONOS
32CASE6: The Failure case.
33CASE7: Check state after control plane failure
34CASE8: Compare topo
35CASE9: Link s3-s28 down
36CASE10: Link s3-s28 up
37CASE11: Switch down
38CASE12: Switch up
39CASE13: Clean up
40CASE14: start election app on all onos nodes
41CASE15: Check that Leadership Election is still functional
42CASE16: Install Distributed Primitives app
43CASE17: Check for basic functionality with distributed primitives
44"""
Jon Hall85794ff2015-07-08 14:12:30 -070045class HAsingleInstanceRestart:
46
47 def __init__( self ):
48 self.default = ''
49
50 def CASE1( self, main ):
51 """
52 CASE1 is to compile ONOS and push it to the test machines
53
54 Startup sequence:
55 cell <name>
56 onos-verify-cell
57 NOTE: temporary - onos-remove-raft-logs
58 onos-uninstall
59 start mininet
60 git pull
61 mvn clean install
62 onos-package
63 onos-install -f
64 onos-wait-for-start
65 start cli sessions
66 start tcpdump
67 """
Jon Halle1a3b752015-07-22 13:02:46 -070068 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080069 import time
Jon Halla440e872016-03-31 15:15:50 -070070 import json
Jon Hall85794ff2015-07-08 14:12:30 -070071 main.log.info( "ONOS Single node cluster restart " +
72 "HA test - initialization" )
73 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070074 main.caseExplanation = "Setup the test environment including " +\
Jon Hall85794ff2015-07-08 14:12:30 -070075 "installing ONOS, starting Mininet and ONOS" +\
76 "cli sessions."
Jon Hall85794ff2015-07-08 14:12:30 -070077
Devin Lim142b5342017-07-20 15:22:39 -070078 # set global variables
Devin Lim58046fa2017-07-05 16:55:00 -070079 # These are for csv plotting in jenkins
80 main.HAlabels = []
81 main.HAdata = []
Jon Halle1a3b752015-07-22 13:02:46 -070082 try:
Devin Lim142b5342017-07-20 15:22:39 -070083 from tests.dependencies.ONOSSetup import ONOSSetup
84 main.testSetUp = ONOSSetup()
85 except ImportError:
86 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070087 main.cleanAndExit()
Devin Lim142b5342017-07-20 15:22:39 -070088 main.testSetUp.envSetupDescription()
89 try:
Jon Hall53c5e662016-04-13 16:06:56 -070090 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070091 main.HA = HA()
Devin Lim142b5342017-07-20 15:22:39 -070092 # load some variables from the params file
93 cellName = main.params[ 'ENV' ][ 'cellName' ]
94 main.apps = main.params[ 'ENV' ][ 'appString' ]
95 main.numCtrls = int( main.params[ 'num_controllers' ] )
96 stepResult = main.testSetUp.envSetup()
97 except Exception as e:
98 main.testSetUp.envSetupException( e )
99 main.testSetUp.evnSetupConclusion( stepResult )
Devin Lim58046fa2017-07-05 16:55:00 -0700100 main.HA.generateGraph( "HAsingleInstanceRestart" )
Devin Lim142b5342017-07-20 15:22:39 -0700101 main.Cluster.setRunningNode( int( main.params[ 'num_controllers' ] ) )
102 ip = main.Cluster.getIps( allNode=True )
103 main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName="SingleHA", removeLog=True,
104 extraApply=[ main.testSetUp.createApplyCell,
105 main.HA.startingMininet,
106 main.testSetUp.createApplyCell ],
Jon Hall4f360bc2017-09-07 10:19:52 -0700107 applyArgs=[ [ main.Cluster, True, cellName, main.Mininet1, True, ip ],
108 None,
109 [ main.Cluster, True, "SingleHA", main.Mininet1,
110 True, main.Cluster.runningNodes[ 0 ].ipAddress ] ] )
Jon Hall85794ff2015-07-08 14:12:30 -0700111
Devin Lim58046fa2017-07-05 16:55:00 -0700112 main.HA.initialSetUp()
Jon Hall9d2dcad2016-04-08 10:15:20 -0700113
Jon Hall85794ff2015-07-08 14:12:30 -0700114 def CASE2( self, main ):
115 """
116 Assign devices to controllers
117 """
Devin Lim58046fa2017-07-05 16:55:00 -0700118 main.HA.assignDevices( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700119
120 def CASE21( self, main ):
121 """
122 Assign mastership to controllers
123 """
Devin Lim58046fa2017-07-05 16:55:00 -0700124 main.HA.assignMastership( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700125
126 def CASE3( self, main ):
127 """
128 Assign intents
129 """
Devin Lim58046fa2017-07-05 16:55:00 -0700130 main.HA.assignIntents( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700131
132 def CASE4( self, main ):
133 """
134 Ping across added host intents
135 """
Jon Hallca319892017-06-15 15:25:22 -0700136 main.HA.pingAcrossHostIntent( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700137
138 def CASE5( self, main ):
139 """
140 Reading state of ONOS
141 """
142 import json
Jon Hall85794ff2015-07-08 14:12:30 -0700143 assert main, "main not defined"
144 assert utilities.assert_equals, "utilities.assert_equals not defined"
145
146 main.case( "Setting up and gathering data for current state" )
147 # The general idea for this test case is to pull the state of
148 # ( intents,flows, topology,... ) from each ONOS node
149 # We can then compare them with each other and also with past states
150
151 main.step( "Check that each switch has a master" )
152 global mastershipState
153 mastershipState = '[]'
154
155 # Assert that each device has a master
Devin Lim142b5342017-07-20 15:22:39 -0700156 main.HA.checkRoleNotNull()
Jon Hall85794ff2015-07-08 14:12:30 -0700157
158 main.step( "Get the Mastership of each switch" )
Devin Lim142b5342017-07-20 15:22:39 -0700159 main.HA.checkTheRole()
Jon Hall85794ff2015-07-08 14:12:30 -0700160
161 main.step( "Get the intents from each controller" )
162 global intentState
163 intentState = []
Devin Lim142b5342017-07-20 15:22:39 -0700164 ONOSIntents = main.Cluster.runningNodes[ 0 ].CLI.intents( jsonFormat=True )
Jon Hall85794ff2015-07-08 14:12:30 -0700165 intentCheck = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700166 if "Error" in ONOSIntents or not ONOSIntents:
Jon Hall85794ff2015-07-08 14:12:30 -0700167 main.log.error( "Error in getting ONOS intents" )
Devin Lim142b5342017-07-20 15:22:39 -0700168 main.log.warn( "ONOS1 intents response: " + repr( ONOSIntents ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700169 else:
170 intentCheck = main.TRUE
171
172 main.step( "Get the flows from each controller" )
173 global flowState
174 flowState = []
175 flowCheck = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700176 ONOSFlows = main.Cluster.runningNodes[ 0 ].CLI.flows( jsonFormat=True )
177 if "Error" in ONOSFlows or not ONOSFlows:
Jon Hall85794ff2015-07-08 14:12:30 -0700178 main.log.error( "Error in getting ONOS flows" )
Devin Lim142b5342017-07-20 15:22:39 -0700179 main.log.warn( "ONOS1 flows repsponse: " + ONOSFlows )
Jon Hall85794ff2015-07-08 14:12:30 -0700180 else:
181 # TODO: Do a better check, maybe compare flows on switches?
Devin Lim142b5342017-07-20 15:22:39 -0700182 flowState = ONOSFlows
Jon Hall85794ff2015-07-08 14:12:30 -0700183 flowCheck = main.TRUE
184
185 main.step( "Get the OF Table entries" )
186 global flows
187 flows = []
188 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -0800189 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700190 if flowCheck == main.FALSE:
191 for table in flows:
192 main.log.warn( table )
193 # TODO: Compare switch flow tables with ONOS flow tables
194
195 main.step( "Collecting topology information from ONOS" )
196 devices = []
Devin Lim142b5342017-07-20 15:22:39 -0700197 devices.append( main.Cluster.runningNodes[ 0 ].CLI.devices() )
Jon Hall85794ff2015-07-08 14:12:30 -0700198 hosts = []
Devin Lim142b5342017-07-20 15:22:39 -0700199 hosts.append( json.loads( main.Cluster.runningNodes[ 0 ].CLI.hosts() ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700200 ports = []
Devin Lim142b5342017-07-20 15:22:39 -0700201 ports.append( main.Cluster.runningNodes[ 0 ].CLI.ports() )
Jon Hall85794ff2015-07-08 14:12:30 -0700202 links = []
Devin Lim142b5342017-07-20 15:22:39 -0700203 links.append( main.Cluster.runningNodes[ 0 ].CLI.links() )
Jon Hall85794ff2015-07-08 14:12:30 -0700204 clusters = []
Devin Lim142b5342017-07-20 15:22:39 -0700205 clusters.append( main.Cluster.runningNodes[ 0 ].CLI.clusters() )
Jon Hall85794ff2015-07-08 14:12:30 -0700206
207 main.step( "Each host has an IP address" )
208 ipResult = main.TRUE
209 for controller in range( 0, len( hosts ) ):
Devin Lim142b5342017-07-20 15:22:39 -0700210 controllerStr = str( main.Cluster.active( controller ) )
Jon Halla440e872016-03-31 15:15:50 -0700211 if hosts[ controller ]:
212 for host in hosts[ controller ]:
Jon Hallf37d44d2017-05-24 10:37:30 -0700213 if not host.get( 'ipAddresses', [] ):
Jon Halla440e872016-03-31 15:15:50 -0700214 main.log.error( "Error with host ips on controller" +
215 controllerStr + ": " + str( host ) )
216 ipResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700217 utilities.assert_equals(
218 expect=main.TRUE,
219 actual=ipResult,
220 onpass="The ips of the hosts aren't empty",
221 onfail="The ip of at least one host is missing" )
222
223 # there should always only be one cluster
224 main.step( "There is only one dataplane cluster" )
225 try:
226 numClusters = len( json.loads( clusters[ 0 ] ) )
227 except ( ValueError, TypeError ):
228 main.log.exception( "Error parsing clusters[0]: " +
229 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -0800230 numClusters = "ERROR"
Jon Hall85794ff2015-07-08 14:12:30 -0700231 clusterResults = main.FALSE
232 if numClusters == 1:
233 clusterResults = main.TRUE
234 utilities.assert_equals(
235 expect=1,
236 actual=numClusters,
237 onpass="ONOS shows 1 SCC",
238 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
239
240 main.step( "Comparing ONOS topology to MN" )
241 devicesResults = main.TRUE
242 linksResults = main.TRUE
243 hostsResults = main.TRUE
244 mnSwitches = main.Mininet1.getSwitches()
245 mnLinks = main.Mininet1.getLinks()
246 mnHosts = main.Mininet1.getHosts()
Devin Lim142b5342017-07-20 15:22:39 -0700247 for controller in main.Cluster.getRunningPos():
248 controllerStr = str( main.Cluster.active( controller ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700249 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -0700250 "Error" not in devices[ controller ] and\
251 "Error" not in ports[ controller ]:
252 currentDevicesResult = main.Mininet1.compareSwitches(
253 mnSwitches,
254 json.loads( devices[ controller ] ),
255 json.loads( ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700256 else:
257 currentDevicesResult = main.FALSE
258 utilities.assert_equals( expect=main.TRUE,
259 actual=currentDevicesResult,
260 onpass="ONOS" + controllerStr +
261 " Switches view is correct",
262 onfail="ONOS" + controllerStr +
263 " Switches view is incorrect" )
264 if links[ controller ] and "Error" not in links[ controller ]:
265 currentLinksResult = main.Mininet1.compareLinks(
266 mnSwitches, mnLinks,
267 json.loads( links[ controller ] ) )
268 else:
269 currentLinksResult = main.FALSE
270 utilities.assert_equals( expect=main.TRUE,
271 actual=currentLinksResult,
272 onpass="ONOS" + controllerStr +
273 " links view is correct",
274 onfail="ONOS" + controllerStr +
275 " links view is incorrect" )
276
Jon Halla440e872016-03-31 15:15:50 -0700277 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall85794ff2015-07-08 14:12:30 -0700278 currentHostsResult = main.Mininet1.compareHosts(
279 mnHosts,
280 hosts[ controller ] )
281 else:
282 currentHostsResult = main.FALSE
283 utilities.assert_equals( expect=main.TRUE,
284 actual=currentHostsResult,
285 onpass="ONOS" + controllerStr +
286 " hosts exist in Mininet",
287 onfail="ONOS" + controllerStr +
288 " hosts don't match Mininet" )
289
290 devicesResults = devicesResults and currentDevicesResult
291 linksResults = linksResults and currentLinksResult
292 hostsResults = hostsResults and currentHostsResult
293
294 main.step( "Device information is correct" )
295 utilities.assert_equals(
296 expect=main.TRUE,
297 actual=devicesResults,
298 onpass="Device information is correct",
299 onfail="Device information is incorrect" )
300
301 main.step( "Links are correct" )
302 utilities.assert_equals(
303 expect=main.TRUE,
304 actual=linksResults,
305 onpass="Link are correct",
306 onfail="Links are incorrect" )
307
308 main.step( "Hosts are correct" )
309 utilities.assert_equals(
310 expect=main.TRUE,
311 actual=hostsResults,
312 onpass="Hosts are correct",
313 onfail="Hosts are incorrect" )
314
Jon Hall8bafdc02017-09-05 11:36:26 -0700315 ONOSMastership, rolesResult, consistentMastership = main.HA.checkTheRole()
316 mastershipState = ONOSMastership[ 0 ]
317
Jon Hall85794ff2015-07-08 14:12:30 -0700318 def CASE6( self, main ):
319 """
320 The Failure case.
321 """
322 import time
Jon Hall85794ff2015-07-08 14:12:30 -0700323 assert main, "main not defined"
324 assert utilities.assert_equals, "utilities.assert_equals not defined"
325
326 # Reset non-persistent variables
327 try:
328 iCounterValue = 0
329 except NameError:
330 main.log.error( "iCounterValue not defined, setting to 0" )
331 iCounterValue = 0
332
333 main.case( "Restart ONOS node" )
Jon Hall783bbf92015-07-23 14:33:19 -0700334 main.caseExplanation = "Killing ONOS process and restart cli " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700335 "sessions once onos is up."
Jon Hall96091e62015-09-21 17:34:17 -0700336
337 main.step( "Checking ONOS Logs for errors" )
Devin Lim142b5342017-07-20 15:22:39 -0700338 for ctrl in main.Cluster.active():
339 main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
340 main.log.warn( main.ONOSbench.checkLogs( ctrl.ip_address ) )
341 ctrl = main.Cluster.runningNodes[ 0 ]
Jon Hall85794ff2015-07-08 14:12:30 -0700342 main.step( "Killing ONOS processes" )
Devin Lim142b5342017-07-20 15:22:39 -0700343 killResult = main.ONOSbench.onosKill( ctrl.ipAddress )
Jon Hall85794ff2015-07-08 14:12:30 -0700344 start = time.time()
345 utilities.assert_equals( expect=main.TRUE, actual=killResult,
346 onpass="ONOS Killed",
347 onfail="Error killing ONOS" )
348
349 main.step( "Checking if ONOS is up yet" )
350 count = 0
351 while count < 10:
Devin Lim142b5342017-07-20 15:22:39 -0700352 onos1Isup = main.ONOSbench.isup( ctrl.ipAddress )
Jon Hall85794ff2015-07-08 14:12:30 -0700353 if onos1Isup == main.TRUE:
354 elapsed = time.time() - start
355 break
356 else:
357 count = count + 1
358 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
359 onpass="ONOS is back up",
360 onfail="ONOS failed to start" )
361
Jon Hall6509dbf2016-06-21 17:01:17 -0700362 main.step( "Starting ONOS CLI sessions" )
Devin Lim142b5342017-07-20 15:22:39 -0700363 cliResults = ctrl.CLI.startOnosCli( ctrl.ipAddress )
Jon Hall85794ff2015-07-08 14:12:30 -0700364 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
365 onpass="ONOS cli startup successful",
366 onfail="ONOS cli startup failed" )
367
368 if elapsed:
369 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
370 str( elapsed ) )
371 main.restartTime = elapsed
372 else:
373 main.restartTime = -1
374 time.sleep( 5 )
375 # rerun on election apps
Devin Lim142b5342017-07-20 15:22:39 -0700376 ctrl.CLI.electionTestRun()
Jon Hall85794ff2015-07-08 14:12:30 -0700377
378 def CASE7( self, main ):
379 """
380 Check state after ONOS failure
381 """
382 import json
Jon Hall85794ff2015-07-08 14:12:30 -0700383 assert main, "main not defined"
384 assert utilities.assert_equals, "utilities.assert_equals not defined"
385 main.case( "Running ONOS Constant State Tests" )
Jon Hall6e709752016-02-01 13:38:46 -0800386
Jon Hall85794ff2015-07-08 14:12:30 -0700387 # Assert that each device has a master
Devin Lim142b5342017-07-20 15:22:39 -0700388 main.HA.checkRoleNotNull()
Jon Hall85794ff2015-07-08 14:12:30 -0700389
390 main.step( "Check if switch roles are consistent across all nodes" )
Jon Hall8bafdc02017-09-05 11:36:26 -0700391 ONOSMastership, rolesResult, consistentMastership = main.HA.checkTheRole()
Devin Lim142b5342017-07-20 15:22:39 -0700392 ONOSMastership = ONOSMastership[ 0 ]
Jon Hall85794ff2015-07-08 14:12:30 -0700393 description2 = "Compare switch roles from before failure"
394 main.step( description2 )
395
Devin Lim142b5342017-07-20 15:22:39 -0700396 currentJson = json.loads( ONOSMastership )
Jon Hall85794ff2015-07-08 14:12:30 -0700397 oldJson = json.loads( mastershipState )
398 mastershipCheck = main.TRUE
399 for i in range( 1, 29 ):
400 switchDPID = str(
401 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
402
403 current = [ switch[ 'master' ] for switch in currentJson
404 if switchDPID in switch[ 'id' ] ]
405 old = [ switch[ 'master' ] for switch in oldJson
406 if switchDPID in switch[ 'id' ] ]
407 if current == old:
408 mastershipCheck = mastershipCheck and main.TRUE
409 else:
Jon Hall8bafdc02017-09-05 11:36:26 -0700410 main.log.warn( "Mastership of switch %s changed; old: %s, new: %s" % ( switchDPID,
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700411 old, current ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700412 mastershipCheck = main.FALSE
413 utilities.assert_equals(
414 expect=main.TRUE,
415 actual=mastershipCheck,
416 onpass="Mastership of Switches was not changed",
417 onfail="Mastership of some switches changed" )
418 mastershipCheck = mastershipCheck and consistentMastership
419
420 main.step( "Get the intents and compare across all nodes" )
Devin Lim142b5342017-07-20 15:22:39 -0700421 ONOSIntents = main.Cluster.runningNodes[ 0 ].CLI.intents( jsonFormat=True )
Jon Hall85794ff2015-07-08 14:12:30 -0700422 intentCheck = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700423 if "Error" in ONOSIntents or not ONOSIntents:
Jon Hall85794ff2015-07-08 14:12:30 -0700424 main.log.error( "Error in getting ONOS intents" )
Devin Lim142b5342017-07-20 15:22:39 -0700425 main.log.warn( "ONOS1 intents response: " + repr( ONOSIntents ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700426 else:
427 intentCheck = main.TRUE
428 utilities.assert_equals(
429 expect=main.TRUE,
430 actual=intentCheck,
431 onpass="Intents are consistent across all ONOS nodes",
432 onfail="ONOS nodes have different views of intents" )
433 # Print the intent states
434 intents = []
Devin Lim142b5342017-07-20 15:22:39 -0700435 intents.append( ONOSIntents )
Jon Hall85794ff2015-07-08 14:12:30 -0700436 intentStates = []
437 for node in intents: # Iter through ONOS nodes
438 nodeStates = []
439 # Iter through intents of a node
440 for intent in json.loads( node ):
441 nodeStates.append( intent[ 'state' ] )
442 intentStates.append( nodeStates )
Jon Hallf37d44d2017-05-24 10:37:30 -0700443 out = [ ( i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
Jon Hall85794ff2015-07-08 14:12:30 -0700444 main.log.info( dict( out ) )
445
446 # NOTE: Store has no durability, so intents are lost across system
447 # restarts
Jon Hall85794ff2015-07-08 14:12:30 -0700448 main.step( "Get the OF Table entries and compare to before " +
449 "component failure" )
450 FlowTables = main.TRUE
Jon Hall85794ff2015-07-08 14:12:30 -0700451 for i in range( 28 ):
452 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -0800453 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hallf37d44d2017-05-24 10:37:30 -0700454 curSwitch = main.Mininet1.flowTableComp( flows[ i ], tmpFlows )
Jon Hall41d39f12016-04-11 22:54:35 -0700455 FlowTables = FlowTables and curSwitch
456 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -0800457 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700458 utilities.assert_equals(
459 expect=main.TRUE,
460 actual=FlowTables,
461 onpass="No changes were found in the flow tables",
462 onfail="Changes were found in the flow tables" )
463
464 main.step( "Leadership Election is still functional" )
465 # Test of LeadershipElection
466
Devin Lim142b5342017-07-20 15:22:39 -0700467 leader = main.Cluster.runningNodes[ 0 ].ipAddress
Jon Hall85794ff2015-07-08 14:12:30 -0700468 leaderResult = main.TRUE
Devin Lim142b5342017-07-20 15:22:39 -0700469 for ctrl in main.Cluster.active():
Jon Hall85794ff2015-07-08 14:12:30 -0700470 # loop through ONOScli handlers
Devin Lim142b5342017-07-20 15:22:39 -0700471 leaderN = ctrl.CLI.electionTestLeader()
Jon Hall85794ff2015-07-08 14:12:30 -0700472 # verify leader is ONOS1
473 # NOTE even though we restarted ONOS, it is the only one so onos 1
474 # must be leader
475 if leaderN == leader:
476 # all is well
477 pass
478 elif leaderN == main.FALSE:
479 # error in response
480 main.log.error( "Something is wrong with " +
481 "electionTestLeader function, check the" +
482 " error logs" )
483 leaderResult = main.FALSE
484 elif leader != leaderN:
485 leaderResult = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700486 main.log.error( ctrl.name + " sees " +
Jon Hall85794ff2015-07-08 14:12:30 -0700487 str( leaderN ) +
488 " as the leader of the election app. " +
489 "Leader should be " + str( leader ) )
490 utilities.assert_equals(
491 expect=main.TRUE,
492 actual=leaderResult,
493 onpass="Leadership election passed",
494 onfail="Something went wrong with Leadership election" )
495
496 def CASE8( self, main ):
497 """
498 Compare topo
499 """
500 import json
501 import time
Jon Hall85794ff2015-07-08 14:12:30 -0700502 assert main, "main not defined"
503 assert utilities.assert_equals, "utilities.assert_equals not defined"
504
505 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -0700506 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall85794ff2015-07-08 14:12:30 -0700507 " and ONOS"
Jon Hall85794ff2015-07-08 14:12:30 -0700508 topoResult = main.FALSE
509 elapsed = 0
510 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -0800511 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall85794ff2015-07-08 14:12:30 -0700512 startTime = time.time()
Devin Lim142b5342017-07-20 15:22:39 -0700513 ctrl = main.Cluster.active( 0 )
Jon Hall85794ff2015-07-08 14:12:30 -0700514 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -0800515 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -0700516 devicesResults = main.TRUE
517 linksResults = main.TRUE
518 hostsResults = main.TRUE
519 hostAttachmentResults = True
Jon Hall85794ff2015-07-08 14:12:30 -0700520 count += 1
521 cliStart = time.time()
522 devices = []
Devin Lim142b5342017-07-20 15:22:39 -0700523 devices.append( ctrl.CLI.devices() )
Jon Hall85794ff2015-07-08 14:12:30 -0700524 hosts = []
Devin Lim142b5342017-07-20 15:22:39 -0700525 hosts.append( json.loads( ctrl.CLI.hosts() ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700526 ipResult = main.TRUE
527 for controller in range( 0, len( hosts ) ):
528 controllerStr = str( controller + 1 )
529 for host in hosts[ controller ]:
530 if host is None or host.get( 'ipAddresses', [] ) == []:
531 main.log.error(
532 "DEBUG:Error with host ips on controller" +
533 controllerStr + ": " + str( host ) )
534 ipResult = main.FALSE
535 ports = []
Devin Lim142b5342017-07-20 15:22:39 -0700536 ports.append( ctrl.CLI.ports() )
Jon Hall85794ff2015-07-08 14:12:30 -0700537 links = []
Devin Lim142b5342017-07-20 15:22:39 -0700538 links.append( ctrl.CLI.links() )
Jon Hall85794ff2015-07-08 14:12:30 -0700539 clusters = []
Devin Lim142b5342017-07-20 15:22:39 -0700540 clusters.append( ctrl.CLI.clusters() )
Jon Hall85794ff2015-07-08 14:12:30 -0700541
542 elapsed = time.time() - startTime
543 cliTime = time.time() - cliStart
544 print "CLI time: " + str( cliTime )
545
546 mnSwitches = main.Mininet1.getSwitches()
547 mnLinks = main.Mininet1.getLinks()
548 mnHosts = main.Mininet1.getHosts()
Devin Lim142b5342017-07-20 15:22:39 -0700549 for controller in main.Cluster.getRunningPos():
550 controllerStr = str( controller )
Jon Hall85794ff2015-07-08 14:12:30 -0700551 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -0700552 "Error" not in devices[ controller ] and\
553 "Error" not in ports[ controller ]:
Jon Hall85794ff2015-07-08 14:12:30 -0700554
Jon Hallc6793552016-01-19 14:18:37 -0800555 try:
556 currentDevicesResult = main.Mininet1.compareSwitches(
557 mnSwitches,
558 json.loads( devices[ controller ] ),
559 json.loads( ports[ controller ] ) )
560 except ( TypeError, ValueError ) as e:
561 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
562 devices[ controller ], ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700563 else:
564 currentDevicesResult = main.FALSE
565 utilities.assert_equals( expect=main.TRUE,
566 actual=currentDevicesResult,
567 onpass="ONOS" + controllerStr +
568 " Switches view is correct",
569 onfail="ONOS" + controllerStr +
570 " Switches view is incorrect" )
571
572 if links[ controller ] and "Error" not in links[ controller ]:
573 currentLinksResult = main.Mininet1.compareLinks(
574 mnSwitches, mnLinks,
575 json.loads( links[ controller ] ) )
576 else:
577 currentLinksResult = main.FALSE
578 utilities.assert_equals( expect=main.TRUE,
579 actual=currentLinksResult,
580 onpass="ONOS" + controllerStr +
581 " links view is correct",
582 onfail="ONOS" + controllerStr +
583 " links view is incorrect" )
584
585 if hosts[ controller ] or "Error" not in hosts[ controller ]:
586 currentHostsResult = main.Mininet1.compareHosts(
587 mnHosts,
588 hosts[ controller ] )
589 else:
590 currentHostsResult = main.FALSE
591 utilities.assert_equals( expect=main.TRUE,
592 actual=currentHostsResult,
593 onpass="ONOS" + controllerStr +
594 " hosts exist in Mininet",
595 onfail="ONOS" + controllerStr +
596 " hosts don't match Mininet" )
597 # CHECKING HOST ATTACHMENT POINTS
598 hostAttachment = True
599 zeroHosts = False
600 # FIXME: topo-HA/obelisk specific mappings:
601 # key is mac and value is dpid
602 mappings = {}
603 for i in range( 1, 29 ): # hosts 1 through 28
604 # set up correct variables:
Jon Hallf37d44d2017-05-24 10:37:30 -0700605 macId = "00:" * 5 + hex( i ).split( "0x" )[ 1 ].upper().zfill( 2 )
Jon Hall85794ff2015-07-08 14:12:30 -0700606 if i == 1:
Jon Hallf37d44d2017-05-24 10:37:30 -0700607 deviceId = "1000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700608 elif i == 2:
Jon Hallf37d44d2017-05-24 10:37:30 -0700609 deviceId = "2000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700610 elif i == 3:
Jon Hallf37d44d2017-05-24 10:37:30 -0700611 deviceId = "3000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700612 elif i == 4:
Jon Hallf37d44d2017-05-24 10:37:30 -0700613 deviceId = "3004".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700614 elif i == 5:
Jon Hallf37d44d2017-05-24 10:37:30 -0700615 deviceId = "5000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700616 elif i == 6:
Jon Hallf37d44d2017-05-24 10:37:30 -0700617 deviceId = "6000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700618 elif i == 7:
Jon Hallf37d44d2017-05-24 10:37:30 -0700619 deviceId = "6007".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700620 elif i >= 8 and i <= 17:
621 dpid = '3' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -0700622 deviceId = dpid.zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700623 elif i >= 18 and i <= 27:
624 dpid = '6' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -0700625 deviceId = dpid.zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700626 elif i == 28:
Jon Hallf37d44d2017-05-24 10:37:30 -0700627 deviceId = "2800".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700628 mappings[ macId ] = deviceId
629 if hosts[ controller ] or "Error" not in hosts[ controller ]:
630 if hosts[ controller ] == []:
631 main.log.warn( "There are no hosts discovered" )
632 zeroHosts = True
633 else:
634 for host in hosts[ controller ]:
635 mac = None
636 location = None
637 device = None
638 port = None
639 try:
640 mac = host.get( 'mac' )
641 assert mac, "mac field could not be found for this host object"
642
Jeremy Ronquillo0e538bc2017-06-13 15:16:09 -0700643 location = host.get( 'locations' )[ 0 ]
Jon Hall85794ff2015-07-08 14:12:30 -0700644 assert location, "location field could not be found for this host object"
645
646 # Trim the protocol identifier off deviceId
Jon Hallf37d44d2017-05-24 10:37:30 -0700647 device = str( location.get( 'elementId' ) ).split( ':' )[ 1 ]
Jon Hall85794ff2015-07-08 14:12:30 -0700648 assert device, "elementId field could not be found for this host location object"
649
650 port = location.get( 'port' )
651 assert port, "port field could not be found for this host location object"
652
653 # Now check if this matches where they should be
654 if mac and device and port:
655 if str( port ) != "1":
656 main.log.error( "The attachment port is incorrect for " +
657 "host " + str( mac ) +
Jon Hallf37d44d2017-05-24 10:37:30 -0700658 ". Expected: 1 Actual: " + str( port ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700659 hostAttachment = False
660 if device != mappings[ str( mac ) ]:
661 main.log.error( "The attachment device is incorrect for " +
662 "host " + str( mac ) +
663 ". Expected: " + mappings[ str( mac ) ] +
664 " Actual: " + device )
665 hostAttachment = False
666 else:
667 hostAttachment = False
668 except AssertionError:
669 main.log.exception( "Json object not as expected" )
670 main.log.error( repr( host ) )
671 hostAttachment = False
672 else:
673 main.log.error( "No hosts json output or \"Error\"" +
674 " in output. hosts = " +
675 repr( hosts[ controller ] ) )
676 if zeroHosts is False:
677 hostAttachment = True
678
Jon Hall85794ff2015-07-08 14:12:30 -0700679 devicesResults = devicesResults and currentDevicesResult
680 linksResults = linksResults and currentLinksResult
681 hostsResults = hostsResults and currentHostsResult
682 hostAttachmentResults = hostAttachmentResults and\
683 hostAttachment
684
Jon Halla440e872016-03-31 15:15:50 -0700685 # "consistent" results don't make sense for single instance
Jon Hall85794ff2015-07-08 14:12:30 -0700686 # there should always only be one cluster
Jon Hall85794ff2015-07-08 14:12:30 -0700687 clusterResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -0700688 try:
689 numClusters = len( json.loads( clusters[ 0 ] ) )
690 except ( ValueError, TypeError ):
691 main.log.exception( "Error parsing clusters[0]: " +
Jon Hallf37d44d2017-05-24 10:37:30 -0700692 repr( clusters[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -0700693 numClusters = "ERROR"
694 clusterResults = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700695 if numClusters == 1:
696 clusterResults = main.TRUE
697 utilities.assert_equals(
698 expect=1,
699 actual=numClusters,
700 onpass="ONOS shows 1 SCC",
701 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
702
703 topoResult = ( devicesResults and linksResults
704 and hostsResults and ipResult and clusterResults and
705 hostAttachmentResults )
706
707 topoResult = topoResult and int( count <= 2 )
708 note = "note it takes about " + str( int( cliTime ) ) + \
709 " seconds for the test to make all the cli calls to fetch " +\
710 "the topology from each ONOS instance"
711 main.log.info(
712 "Very crass estimate for topology discovery/convergence( " +
713 str( note ) + " ): " + str( elapsed ) + " seconds, " +
714 str( count ) + " tries" )
715 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
716 onpass="Topology Check Test successful",
717 onfail="Topology Check Test NOT successful" )
Jon Hall41d39f12016-04-11 22:54:35 -0700718 main.step( "Checking ONOS nodes" )
719 nodeResults = utilities.retry( main.HA.nodesCheck,
720 False,
Devin Lim142b5342017-07-20 15:22:39 -0700721 args=[ main.Cluster.active() ],
Jon Hall41d39f12016-04-11 22:54:35 -0700722 attempts=5 )
723
724 utilities.assert_equals( expect=True, actual=nodeResults,
725 onpass="Nodes check successful",
726 onfail="Nodes check NOT successful" )
727 if not nodeResults:
Devin Lim142b5342017-07-20 15:22:39 -0700728 for ctrl in main.Cluster.active():
Jon Hall41d39f12016-04-11 22:54:35 -0700729 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700730 ctrl.name,
731 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700732
Jon Halld2871c22016-07-26 11:01:14 -0700733 if not topoResult:
Devin Lim44075962017-08-11 10:56:37 -0700734 main.cleanAndExit()
Jon Halld2871c22016-07-26 11:01:14 -0700735
Jon Hall85794ff2015-07-08 14:12:30 -0700736 def CASE9( self, main ):
737 """
738 Link s3-s28 down
739 """
Devin Lim58046fa2017-07-05 16:55:00 -0700740 main.HA.linkDown( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700741
742 def CASE10( self, main ):
743 """
744 Link s3-s28 up
745 """
Devin Lim58046fa2017-07-05 16:55:00 -0700746 main.HA.linkUp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700747
748 def CASE11( self, main ):
749 """
750 Switch Down
751 """
752 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700753 main.HA.switchDown( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700754
755 def CASE12( self, main ):
756 """
757 Switch Up
758 """
759 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700760 main.HA.switchUp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700761
762 def CASE13( self, main ):
763 """
764 Clean up
765 """
Devin Lim58046fa2017-07-05 16:55:00 -0700766 main.HAlabels.append( "Restart" )
767 main.HAdata.append( str( main.restartTime ) )
768 main.HA.cleanUp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700769
770 def CASE14( self, main ):
771 """
772 start election app on all onos nodes
773 """
Devin Lim58046fa2017-07-05 16:55:00 -0700774 main.HA.startElectionApp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700775
776 def CASE15( self, main ):
777 """
778 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -0700779 15.1 Run election on each node
780 15.2 Check that each node has the same leaders and candidates
781 15.3 Find current leader and withdraw
782 15.4 Check that a new node was elected leader
783 15.5 Check that that new leader was the candidate of old leader
784 15.6 Run for election on old leader
785 15.7 Check that oldLeader is a candidate, and leader if only 1 node
786 15.8 Make sure that the old leader was added to the candidate list
787
788 old and new variable prefixes refer to data from before vs after
789 withdrawl and later before withdrawl vs after re-election
Jon Hall85794ff2015-07-08 14:12:30 -0700790 """
Devin Lim58046fa2017-07-05 16:55:00 -0700791 main.HA.isElectionFunctional( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700792
793 def CASE16( self, main ):
794 """
795 Install Distributed Primitives app
796 """
Devin Lim58046fa2017-07-05 16:55:00 -0700797 main.HA.installDistributedPrimitiveApp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700798
799 def CASE17( self, main ):
800 """
801 Check for basic functionality with distributed primitives
802 """
Devin Lim58046fa2017-07-05 16:55:00 -0700803 main.HA.checkDistPrimitivesFunc( main )