blob: f8d5a49451dfbd4877e691c3be713f684e0b1ddd [file] [log] [blame]
Jon Hall85794ff2015-07-08 14:12:30 -07001"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07002Copyright 2015 Open Networking Foundation (ONF)
3
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
11 (at your option) any later version.
12
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
21
22"""
Jon Hall85794ff2015-07-08 14:12:30 -070023Description: This test is to determine if a single
24 instance ONOS 'cluster' can handle a restart
25
26List of test cases:
27CASE1: Compile ONOS and push it to the test machines
28CASE2: Assign devices to controllers
29CASE21: Assign mastership to controllers
30CASE3: Assign intents
31CASE4: Ping across added host intents
32CASE5: Reading state of ONOS
33CASE6: The Failure case.
34CASE7: Check state after control plane failure
35CASE8: Compare topo
36CASE9: Link s3-s28 down
37CASE10: Link s3-s28 up
38CASE11: Switch down
39CASE12: Switch up
40CASE13: Clean up
41CASE14: start election app on all onos nodes
42CASE15: Check that Leadership Election is still functional
43CASE16: Install Distributed Primitives app
44CASE17: Check for basic functionality with distributed primitives
45"""
Jon Hall85794ff2015-07-08 14:12:30 -070046class HAsingleInstanceRestart:
47
48 def __init__( self ):
49 self.default = ''
50
51 def CASE1( self, main ):
52 """
53 CASE1 is to compile ONOS and push it to the test machines
54
55 Startup sequence:
56 cell <name>
57 onos-verify-cell
58 NOTE: temporary - onos-remove-raft-logs
59 onos-uninstall
60 start mininet
61 git pull
62 mvn clean install
63 onos-package
64 onos-install -f
65 onos-wait-for-start
66 start cli sessions
67 start tcpdump
68 """
Jon Halle1a3b752015-07-22 13:02:46 -070069 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080070 import time
Jon Halla440e872016-03-31 15:15:50 -070071 import json
Jon Hall85794ff2015-07-08 14:12:30 -070072 main.log.info( "ONOS Single node cluster restart " +
73 "HA test - initialization" )
74 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070075 main.caseExplanation = "Setup the test environment including " +\
Jon Hall85794ff2015-07-08 14:12:30 -070076 "installing ONOS, starting Mininet and ONOS" +\
77 "cli sessions."
Jon Hall85794ff2015-07-08 14:12:30 -070078
Devin Lim142b5342017-07-20 15:22:39 -070079 # set global variables
Devin Lim58046fa2017-07-05 16:55:00 -070080 # These are for csv plotting in jenkins
81 main.HAlabels = []
82 main.HAdata = []
Jon Halle1a3b752015-07-22 13:02:46 -070083 try:
Devin Lim142b5342017-07-20 15:22:39 -070084 from tests.dependencies.ONOSSetup import ONOSSetup
85 main.testSetUp = ONOSSetup()
86 except ImportError:
87 main.log.error( "ONOSSetup not found. exiting the test" )
88 main.exit()
89 main.testSetUp.envSetupDescription()
90 try:
Jon Hall53c5e662016-04-13 16:06:56 -070091 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070092 main.HA = HA()
Devin Lim142b5342017-07-20 15:22:39 -070093 # load some variables from the params file
94 cellName = main.params[ 'ENV' ][ 'cellName' ]
95 main.apps = main.params[ 'ENV' ][ 'appString' ]
96 main.numCtrls = int( main.params[ 'num_controllers' ] )
97 stepResult = main.testSetUp.envSetup()
98 except Exception as e:
99 main.testSetUp.envSetupException( e )
100 main.testSetUp.evnSetupConclusion( stepResult )
Devin Lim58046fa2017-07-05 16:55:00 -0700101 main.HA.generateGraph( "HAsingleInstanceRestart" )
Devin Lim142b5342017-07-20 15:22:39 -0700102 main.Cluster.setRunningNode( int( main.params[ 'num_controllers' ] ) )
103 ip = main.Cluster.getIps( allNode=True )
104 main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName="SingleHA", removeLog=True,
105 extraApply=[ main.testSetUp.createApplyCell,
106 main.HA.startingMininet,
107 main.testSetUp.createApplyCell ],
108 arg=[ [ main.Cluster, True, cellName, main.Mininet1, True, ip ],
109 None,
110 [ main.Cluster, True, "SingleHA", main.Mininet1,
111 True, main.Cluster.runningNodes[ 0 ].ipAddress ] ] )
Jon Hall85794ff2015-07-08 14:12:30 -0700112
Devin Lim58046fa2017-07-05 16:55:00 -0700113 main.HA.initialSetUp()
Jon Hall9d2dcad2016-04-08 10:15:20 -0700114
Jon Hall85794ff2015-07-08 14:12:30 -0700115 def CASE2( self, main ):
116 """
117 Assign devices to controllers
118 """
Devin Lim58046fa2017-07-05 16:55:00 -0700119 main.HA.assignDevices( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700120
121 def CASE21( self, main ):
122 """
123 Assign mastership to controllers
124 """
Devin Lim58046fa2017-07-05 16:55:00 -0700125 main.HA.assignMastership( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700126
127 def CASE3( self, main ):
128 """
129 Assign intents
130 """
Devin Lim58046fa2017-07-05 16:55:00 -0700131 main.HA.assignIntents( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700132
133 def CASE4( self, main ):
134 """
135 Ping across added host intents
136 """
Jon Hallca319892017-06-15 15:25:22 -0700137 main.HA.pingAcrossHostIntent( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700138
139 def CASE5( self, main ):
140 """
141 Reading state of ONOS
142 """
143 import json
Jon Hall85794ff2015-07-08 14:12:30 -0700144 assert main, "main not defined"
145 assert utilities.assert_equals, "utilities.assert_equals not defined"
146
147 main.case( "Setting up and gathering data for current state" )
148 # The general idea for this test case is to pull the state of
149 # ( intents,flows, topology,... ) from each ONOS node
150 # We can then compare them with each other and also with past states
151
152 main.step( "Check that each switch has a master" )
153 global mastershipState
154 mastershipState = '[]'
155
156 # Assert that each device has a master
Devin Lim142b5342017-07-20 15:22:39 -0700157 main.HA.checkRoleNotNull()
Jon Hall85794ff2015-07-08 14:12:30 -0700158
159 main.step( "Get the Mastership of each switch" )
Devin Lim142b5342017-07-20 15:22:39 -0700160 main.HA.checkTheRole()
Jon Hall85794ff2015-07-08 14:12:30 -0700161
162 main.step( "Get the intents from each controller" )
163 global intentState
164 intentState = []
Devin Lim142b5342017-07-20 15:22:39 -0700165 ONOSIntents = main.Cluster.runningNodes[ 0 ].CLI.intents( jsonFormat=True )
Jon Hall85794ff2015-07-08 14:12:30 -0700166 intentCheck = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700167 if "Error" in ONOSIntents or not ONOSIntents:
Jon Hall85794ff2015-07-08 14:12:30 -0700168 main.log.error( "Error in getting ONOS intents" )
Devin Lim142b5342017-07-20 15:22:39 -0700169 main.log.warn( "ONOS1 intents response: " + repr( ONOSIntents ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700170 else:
171 intentCheck = main.TRUE
172
173 main.step( "Get the flows from each controller" )
174 global flowState
175 flowState = []
176 flowCheck = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700177 ONOSFlows = main.Cluster.runningNodes[ 0 ].CLI.flows( jsonFormat=True )
178 if "Error" in ONOSFlows or not ONOSFlows:
Jon Hall85794ff2015-07-08 14:12:30 -0700179 main.log.error( "Error in getting ONOS flows" )
Devin Lim142b5342017-07-20 15:22:39 -0700180 main.log.warn( "ONOS1 flows repsponse: " + ONOSFlows )
Jon Hall85794ff2015-07-08 14:12:30 -0700181 else:
182 # TODO: Do a better check, maybe compare flows on switches?
Devin Lim142b5342017-07-20 15:22:39 -0700183 flowState = ONOSFlows
Jon Hall85794ff2015-07-08 14:12:30 -0700184 flowCheck = main.TRUE
185
186 main.step( "Get the OF Table entries" )
187 global flows
188 flows = []
189 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -0800190 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700191 if flowCheck == main.FALSE:
192 for table in flows:
193 main.log.warn( table )
194 # TODO: Compare switch flow tables with ONOS flow tables
195
196 main.step( "Collecting topology information from ONOS" )
197 devices = []
Devin Lim142b5342017-07-20 15:22:39 -0700198 devices.append( main.Cluster.runningNodes[ 0 ].CLI.devices() )
Jon Hall85794ff2015-07-08 14:12:30 -0700199 hosts = []
Devin Lim142b5342017-07-20 15:22:39 -0700200 hosts.append( json.loads( main.Cluster.runningNodes[ 0 ].CLI.hosts() ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700201 ports = []
Devin Lim142b5342017-07-20 15:22:39 -0700202 ports.append( main.Cluster.runningNodes[ 0 ].CLI.ports() )
Jon Hall85794ff2015-07-08 14:12:30 -0700203 links = []
Devin Lim142b5342017-07-20 15:22:39 -0700204 links.append( main.Cluster.runningNodes[ 0 ].CLI.links() )
Jon Hall85794ff2015-07-08 14:12:30 -0700205 clusters = []
Devin Lim142b5342017-07-20 15:22:39 -0700206 clusters.append( main.Cluster.runningNodes[ 0 ].CLI.clusters() )
Jon Hall85794ff2015-07-08 14:12:30 -0700207
208 main.step( "Each host has an IP address" )
209 ipResult = main.TRUE
210 for controller in range( 0, len( hosts ) ):
Devin Lim142b5342017-07-20 15:22:39 -0700211 controllerStr = str( main.Cluster.active( controller ) )
Jon Halla440e872016-03-31 15:15:50 -0700212 if hosts[ controller ]:
213 for host in hosts[ controller ]:
Jon Hallf37d44d2017-05-24 10:37:30 -0700214 if not host.get( 'ipAddresses', [] ):
Jon Halla440e872016-03-31 15:15:50 -0700215 main.log.error( "Error with host ips on controller" +
216 controllerStr + ": " + str( host ) )
217 ipResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700218 utilities.assert_equals(
219 expect=main.TRUE,
220 actual=ipResult,
221 onpass="The ips of the hosts aren't empty",
222 onfail="The ip of at least one host is missing" )
223
224 # there should always only be one cluster
225 main.step( "There is only one dataplane cluster" )
226 try:
227 numClusters = len( json.loads( clusters[ 0 ] ) )
228 except ( ValueError, TypeError ):
229 main.log.exception( "Error parsing clusters[0]: " +
230 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -0800231 numClusters = "ERROR"
Jon Hall85794ff2015-07-08 14:12:30 -0700232 clusterResults = main.FALSE
233 if numClusters == 1:
234 clusterResults = main.TRUE
235 utilities.assert_equals(
236 expect=1,
237 actual=numClusters,
238 onpass="ONOS shows 1 SCC",
239 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
240
241 main.step( "Comparing ONOS topology to MN" )
242 devicesResults = main.TRUE
243 linksResults = main.TRUE
244 hostsResults = main.TRUE
245 mnSwitches = main.Mininet1.getSwitches()
246 mnLinks = main.Mininet1.getLinks()
247 mnHosts = main.Mininet1.getHosts()
Devin Lim142b5342017-07-20 15:22:39 -0700248 for controller in main.Cluster.getRunningPos():
249 controllerStr = str( main.Cluster.active( controller ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700250 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -0700251 "Error" not in devices[ controller ] and\
252 "Error" not in ports[ controller ]:
253 currentDevicesResult = main.Mininet1.compareSwitches(
254 mnSwitches,
255 json.loads( devices[ controller ] ),
256 json.loads( ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700257 else:
258 currentDevicesResult = main.FALSE
259 utilities.assert_equals( expect=main.TRUE,
260 actual=currentDevicesResult,
261 onpass="ONOS" + controllerStr +
262 " Switches view is correct",
263 onfail="ONOS" + controllerStr +
264 " Switches view is incorrect" )
265 if links[ controller ] and "Error" not in links[ controller ]:
266 currentLinksResult = main.Mininet1.compareLinks(
267 mnSwitches, mnLinks,
268 json.loads( links[ controller ] ) )
269 else:
270 currentLinksResult = main.FALSE
271 utilities.assert_equals( expect=main.TRUE,
272 actual=currentLinksResult,
273 onpass="ONOS" + controllerStr +
274 " links view is correct",
275 onfail="ONOS" + controllerStr +
276 " links view is incorrect" )
277
Jon Halla440e872016-03-31 15:15:50 -0700278 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall85794ff2015-07-08 14:12:30 -0700279 currentHostsResult = main.Mininet1.compareHosts(
280 mnHosts,
281 hosts[ controller ] )
282 else:
283 currentHostsResult = main.FALSE
284 utilities.assert_equals( expect=main.TRUE,
285 actual=currentHostsResult,
286 onpass="ONOS" + controllerStr +
287 " hosts exist in Mininet",
288 onfail="ONOS" + controllerStr +
289 " hosts don't match Mininet" )
290
291 devicesResults = devicesResults and currentDevicesResult
292 linksResults = linksResults and currentLinksResult
293 hostsResults = hostsResults and currentHostsResult
294
295 main.step( "Device information is correct" )
296 utilities.assert_equals(
297 expect=main.TRUE,
298 actual=devicesResults,
299 onpass="Device information is correct",
300 onfail="Device information is incorrect" )
301
302 main.step( "Links are correct" )
303 utilities.assert_equals(
304 expect=main.TRUE,
305 actual=linksResults,
306 onpass="Link are correct",
307 onfail="Links are incorrect" )
308
309 main.step( "Hosts are correct" )
310 utilities.assert_equals(
311 expect=main.TRUE,
312 actual=hostsResults,
313 onpass="Hosts are correct",
314 onfail="Hosts are incorrect" )
315
316 def CASE6( self, main ):
317 """
318 The Failure case.
319 """
320 import time
Jon Hall85794ff2015-07-08 14:12:30 -0700321 assert main, "main not defined"
322 assert utilities.assert_equals, "utilities.assert_equals not defined"
323
324 # Reset non-persistent variables
325 try:
326 iCounterValue = 0
327 except NameError:
328 main.log.error( "iCounterValue not defined, setting to 0" )
329 iCounterValue = 0
330
331 main.case( "Restart ONOS node" )
Jon Hall783bbf92015-07-23 14:33:19 -0700332 main.caseExplanation = "Killing ONOS process and restart cli " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700333 "sessions once onos is up."
Jon Hall96091e62015-09-21 17:34:17 -0700334
335 main.step( "Checking ONOS Logs for errors" )
Devin Lim142b5342017-07-20 15:22:39 -0700336 for ctrl in main.Cluster.active():
337 main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
338 main.log.warn( main.ONOSbench.checkLogs( ctrl.ip_address ) )
339 ctrl = main.Cluster.runningNodes[ 0 ]
Jon Hall85794ff2015-07-08 14:12:30 -0700340 main.step( "Killing ONOS processes" )
Devin Lim142b5342017-07-20 15:22:39 -0700341 killResult = main.ONOSbench.onosKill( ctrl.ipAddress )
Jon Hall85794ff2015-07-08 14:12:30 -0700342 start = time.time()
343 utilities.assert_equals( expect=main.TRUE, actual=killResult,
344 onpass="ONOS Killed",
345 onfail="Error killing ONOS" )
346
347 main.step( "Checking if ONOS is up yet" )
348 count = 0
349 while count < 10:
Devin Lim142b5342017-07-20 15:22:39 -0700350 onos1Isup = main.ONOSbench.isup( ctrl.ipAddress )
Jon Hall85794ff2015-07-08 14:12:30 -0700351 if onos1Isup == main.TRUE:
352 elapsed = time.time() - start
353 break
354 else:
355 count = count + 1
356 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
357 onpass="ONOS is back up",
358 onfail="ONOS failed to start" )
359
Jon Hall6509dbf2016-06-21 17:01:17 -0700360 main.step( "Starting ONOS CLI sessions" )
Devin Lim142b5342017-07-20 15:22:39 -0700361 cliResults = ctrl.CLI.startOnosCli( ctrl.ipAddress )
Jon Hall85794ff2015-07-08 14:12:30 -0700362 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
363 onpass="ONOS cli startup successful",
364 onfail="ONOS cli startup failed" )
365
366 if elapsed:
367 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
368 str( elapsed ) )
369 main.restartTime = elapsed
370 else:
371 main.restartTime = -1
372 time.sleep( 5 )
373 # rerun on election apps
Devin Lim142b5342017-07-20 15:22:39 -0700374 ctrl.CLI.electionTestRun()
Jon Hall85794ff2015-07-08 14:12:30 -0700375
376 def CASE7( self, main ):
377 """
378 Check state after ONOS failure
379 """
380 import json
Jon Hall85794ff2015-07-08 14:12:30 -0700381 assert main, "main not defined"
382 assert utilities.assert_equals, "utilities.assert_equals not defined"
383 main.case( "Running ONOS Constant State Tests" )
Jon Hall6e709752016-02-01 13:38:46 -0800384
Jon Hall85794ff2015-07-08 14:12:30 -0700385 # Assert that each device has a master
Devin Lim142b5342017-07-20 15:22:39 -0700386 main.HA.checkRoleNotNull()
Jon Hall85794ff2015-07-08 14:12:30 -0700387
388 main.step( "Check if switch roles are consistent across all nodes" )
Devin Lim142b5342017-07-20 15:22:39 -0700389 ONOSMastership, rolesResult,consistentMastership = main.HA.checkTheRole()
390 ONOSMastership = ONOSMastership[ 0 ]
Jon Hall85794ff2015-07-08 14:12:30 -0700391 description2 = "Compare switch roles from before failure"
392 main.step( description2 )
393
Devin Lim142b5342017-07-20 15:22:39 -0700394 currentJson = json.loads( ONOSMastership )
Jon Hall85794ff2015-07-08 14:12:30 -0700395 oldJson = json.loads( mastershipState )
396 mastershipCheck = main.TRUE
397 for i in range( 1, 29 ):
398 switchDPID = str(
399 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
400
401 current = [ switch[ 'master' ] for switch in currentJson
402 if switchDPID in switch[ 'id' ] ]
403 old = [ switch[ 'master' ] for switch in oldJson
404 if switchDPID in switch[ 'id' ] ]
405 if current == old:
406 mastershipCheck = mastershipCheck and main.TRUE
407 else:
408 main.log.warn( "Mastership of switch %s changed" % switchDPID )
409 mastershipCheck = main.FALSE
410 utilities.assert_equals(
411 expect=main.TRUE,
412 actual=mastershipCheck,
413 onpass="Mastership of Switches was not changed",
414 onfail="Mastership of some switches changed" )
415 mastershipCheck = mastershipCheck and consistentMastership
416
417 main.step( "Get the intents and compare across all nodes" )
Devin Lim142b5342017-07-20 15:22:39 -0700418 ONOSIntents = main.Cluster.runningNodes[ 0 ].CLI.intents( jsonFormat=True )
Jon Hall85794ff2015-07-08 14:12:30 -0700419 intentCheck = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700420 if "Error" in ONOSIntents or not ONOSIntents:
Jon Hall85794ff2015-07-08 14:12:30 -0700421 main.log.error( "Error in getting ONOS intents" )
Devin Lim142b5342017-07-20 15:22:39 -0700422 main.log.warn( "ONOS1 intents response: " + repr( ONOSIntents ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700423 else:
424 intentCheck = main.TRUE
425 utilities.assert_equals(
426 expect=main.TRUE,
427 actual=intentCheck,
428 onpass="Intents are consistent across all ONOS nodes",
429 onfail="ONOS nodes have different views of intents" )
430 # Print the intent states
431 intents = []
Devin Lim142b5342017-07-20 15:22:39 -0700432 intents.append( ONOSIntents )
Jon Hall85794ff2015-07-08 14:12:30 -0700433 intentStates = []
434 for node in intents: # Iter through ONOS nodes
435 nodeStates = []
436 # Iter through intents of a node
437 for intent in json.loads( node ):
438 nodeStates.append( intent[ 'state' ] )
439 intentStates.append( nodeStates )
Jon Hallf37d44d2017-05-24 10:37:30 -0700440 out = [ ( i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
Jon Hall85794ff2015-07-08 14:12:30 -0700441 main.log.info( dict( out ) )
442
443 # NOTE: Store has no durability, so intents are lost across system
444 # restarts
445 """
446 main.step( "Compare current intents with intents before the failure" )
447 # NOTE: this requires case 5 to pass for intentState to be set.
448 # maybe we should stop the test if that fails?
449 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -0700450 try:
451 intentState
452 except NameError:
453 main.log.warn( "No previous intent state was saved" )
454 else:
455 if intentState and intentState == ONOSIntents[ 0 ]:
456 sameIntents = main.TRUE
457 main.log.info( "Intents are consistent with before failure" )
458 # TODO: possibly the states have changed? we may need to figure out
459 # what the acceptable states are
460 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
461 sameIntents = main.TRUE
462 try:
463 before = json.loads( intentState )
464 after = json.loads( ONOSIntents[ 0 ] )
465 for intent in before:
466 if intent not in after:
467 sameIntents = main.FALSE
468 main.log.debug( "Intent is not currently in ONOS " +
469 "(at least in the same form):" )
470 main.log.debug( json.dumps( intent ) )
471 except ( ValueError, TypeError ):
472 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700473 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -0700474 main.log.debug( repr( intentState ) )
475 if sameIntents == main.FALSE:
476 try:
477 main.log.debug( "ONOS intents before: " )
478 main.log.debug( json.dumps( json.loads( intentState ),
479 sort_keys=True, indent=4,
480 separators=( ',', ': ' ) ) )
481 main.log.debug( "Current ONOS intents: " )
482 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
483 sort_keys=True, indent=4,
484 separators=( ',', ': ' ) ) )
485 except ( ValueError, TypeError ):
486 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700487 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -0700488 main.log.debug( repr( intentState ) )
489 utilities.assert_equals(
490 expect=main.TRUE,
491 actual=sameIntents,
492 onpass="Intents are consistent with before failure",
493 onfail="The Intents changed during failure" )
Jon Hall85794ff2015-07-08 14:12:30 -0700494 intentCheck = intentCheck and sameIntents
495 """
496 main.step( "Get the OF Table entries and compare to before " +
497 "component failure" )
498 FlowTables = main.TRUE
Jon Hall85794ff2015-07-08 14:12:30 -0700499 for i in range( 28 ):
500 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -0800501 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hallf37d44d2017-05-24 10:37:30 -0700502 curSwitch = main.Mininet1.flowTableComp( flows[ i ], tmpFlows )
Jon Hall41d39f12016-04-11 22:54:35 -0700503 FlowTables = FlowTables and curSwitch
504 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -0800505 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700506 utilities.assert_equals(
507 expect=main.TRUE,
508 actual=FlowTables,
509 onpass="No changes were found in the flow tables",
510 onfail="Changes were found in the flow tables" )
511
512 main.step( "Leadership Election is still functional" )
513 # Test of LeadershipElection
514
Devin Lim142b5342017-07-20 15:22:39 -0700515 leader = main.Cluster.runningNodes[ 0 ].ipAddress
Jon Hall85794ff2015-07-08 14:12:30 -0700516 leaderResult = main.TRUE
Devin Lim142b5342017-07-20 15:22:39 -0700517 for ctrl in main.Cluster.active():
Jon Hall85794ff2015-07-08 14:12:30 -0700518 # loop through ONOScli handlers
Devin Lim142b5342017-07-20 15:22:39 -0700519 leaderN = ctrl.CLI.electionTestLeader()
Jon Hall85794ff2015-07-08 14:12:30 -0700520 # verify leader is ONOS1
521 # NOTE even though we restarted ONOS, it is the only one so onos 1
522 # must be leader
523 if leaderN == leader:
524 # all is well
525 pass
526 elif leaderN == main.FALSE:
527 # error in response
528 main.log.error( "Something is wrong with " +
529 "electionTestLeader function, check the" +
530 " error logs" )
531 leaderResult = main.FALSE
532 elif leader != leaderN:
533 leaderResult = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700534 main.log.error( ctrl.name + " sees " +
Jon Hall85794ff2015-07-08 14:12:30 -0700535 str( leaderN ) +
536 " as the leader of the election app. " +
537 "Leader should be " + str( leader ) )
538 utilities.assert_equals(
539 expect=main.TRUE,
540 actual=leaderResult,
541 onpass="Leadership election passed",
542 onfail="Something went wrong with Leadership election" )
543
544 def CASE8( self, main ):
545 """
546 Compare topo
547 """
548 import json
549 import time
Jon Hall85794ff2015-07-08 14:12:30 -0700550 assert main, "main not defined"
551 assert utilities.assert_equals, "utilities.assert_equals not defined"
552
553 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -0700554 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall85794ff2015-07-08 14:12:30 -0700555 " and ONOS"
Jon Hall85794ff2015-07-08 14:12:30 -0700556 topoResult = main.FALSE
557 elapsed = 0
558 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -0800559 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall85794ff2015-07-08 14:12:30 -0700560 startTime = time.time()
Devin Lim142b5342017-07-20 15:22:39 -0700561 ctrl = main.Cluster.active( 0 )
Jon Hall85794ff2015-07-08 14:12:30 -0700562 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -0800563 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -0700564 devicesResults = main.TRUE
565 linksResults = main.TRUE
566 hostsResults = main.TRUE
567 hostAttachmentResults = True
Jon Hall85794ff2015-07-08 14:12:30 -0700568 count += 1
569 cliStart = time.time()
570 devices = []
Devin Lim142b5342017-07-20 15:22:39 -0700571 devices.append( ctrl.CLI.devices() )
Jon Hall85794ff2015-07-08 14:12:30 -0700572 hosts = []
Devin Lim142b5342017-07-20 15:22:39 -0700573 hosts.append( json.loads( ctrl.CLI.hosts() ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700574 ipResult = main.TRUE
575 for controller in range( 0, len( hosts ) ):
576 controllerStr = str( controller + 1 )
577 for host in hosts[ controller ]:
578 if host is None or host.get( 'ipAddresses', [] ) == []:
579 main.log.error(
580 "DEBUG:Error with host ips on controller" +
581 controllerStr + ": " + str( host ) )
582 ipResult = main.FALSE
583 ports = []
Devin Lim142b5342017-07-20 15:22:39 -0700584 ports.append( ctrl.CLI.ports() )
Jon Hall85794ff2015-07-08 14:12:30 -0700585 links = []
Devin Lim142b5342017-07-20 15:22:39 -0700586 links.append( ctrl.CLI.links() )
Jon Hall85794ff2015-07-08 14:12:30 -0700587 clusters = []
Devin Lim142b5342017-07-20 15:22:39 -0700588 clusters.append( ctrl.CLI.clusters() )
Jon Hall85794ff2015-07-08 14:12:30 -0700589
590 elapsed = time.time() - startTime
591 cliTime = time.time() - cliStart
592 print "CLI time: " + str( cliTime )
593
594 mnSwitches = main.Mininet1.getSwitches()
595 mnLinks = main.Mininet1.getLinks()
596 mnHosts = main.Mininet1.getHosts()
Devin Lim142b5342017-07-20 15:22:39 -0700597 for controller in main.Cluster.getRunningPos():
598 controllerStr = str( controller )
Jon Hall85794ff2015-07-08 14:12:30 -0700599 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -0700600 "Error" not in devices[ controller ] and\
601 "Error" not in ports[ controller ]:
Jon Hall85794ff2015-07-08 14:12:30 -0700602
Jon Hallc6793552016-01-19 14:18:37 -0800603 try:
604 currentDevicesResult = main.Mininet1.compareSwitches(
605 mnSwitches,
606 json.loads( devices[ controller ] ),
607 json.loads( ports[ controller ] ) )
608 except ( TypeError, ValueError ) as e:
609 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
610 devices[ controller ], ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700611 else:
612 currentDevicesResult = main.FALSE
613 utilities.assert_equals( expect=main.TRUE,
614 actual=currentDevicesResult,
615 onpass="ONOS" + controllerStr +
616 " Switches view is correct",
617 onfail="ONOS" + controllerStr +
618 " Switches view is incorrect" )
619
620 if links[ controller ] and "Error" not in links[ controller ]:
621 currentLinksResult = main.Mininet1.compareLinks(
622 mnSwitches, mnLinks,
623 json.loads( links[ controller ] ) )
624 else:
625 currentLinksResult = main.FALSE
626 utilities.assert_equals( expect=main.TRUE,
627 actual=currentLinksResult,
628 onpass="ONOS" + controllerStr +
629 " links view is correct",
630 onfail="ONOS" + controllerStr +
631 " links view is incorrect" )
632
633 if hosts[ controller ] or "Error" not in hosts[ controller ]:
634 currentHostsResult = main.Mininet1.compareHosts(
635 mnHosts,
636 hosts[ controller ] )
637 else:
638 currentHostsResult = main.FALSE
639 utilities.assert_equals( expect=main.TRUE,
640 actual=currentHostsResult,
641 onpass="ONOS" + controllerStr +
642 " hosts exist in Mininet",
643 onfail="ONOS" + controllerStr +
644 " hosts don't match Mininet" )
645 # CHECKING HOST ATTACHMENT POINTS
646 hostAttachment = True
647 zeroHosts = False
648 # FIXME: topo-HA/obelisk specific mappings:
649 # key is mac and value is dpid
650 mappings = {}
651 for i in range( 1, 29 ): # hosts 1 through 28
652 # set up correct variables:
Jon Hallf37d44d2017-05-24 10:37:30 -0700653 macId = "00:" * 5 + hex( i ).split( "0x" )[ 1 ].upper().zfill( 2 )
Jon Hall85794ff2015-07-08 14:12:30 -0700654 if i == 1:
Jon Hallf37d44d2017-05-24 10:37:30 -0700655 deviceId = "1000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700656 elif i == 2:
Jon Hallf37d44d2017-05-24 10:37:30 -0700657 deviceId = "2000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700658 elif i == 3:
Jon Hallf37d44d2017-05-24 10:37:30 -0700659 deviceId = "3000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700660 elif i == 4:
Jon Hallf37d44d2017-05-24 10:37:30 -0700661 deviceId = "3004".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700662 elif i == 5:
Jon Hallf37d44d2017-05-24 10:37:30 -0700663 deviceId = "5000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700664 elif i == 6:
Jon Hallf37d44d2017-05-24 10:37:30 -0700665 deviceId = "6000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700666 elif i == 7:
Jon Hallf37d44d2017-05-24 10:37:30 -0700667 deviceId = "6007".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700668 elif i >= 8 and i <= 17:
669 dpid = '3' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -0700670 deviceId = dpid.zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700671 elif i >= 18 and i <= 27:
672 dpid = '6' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -0700673 deviceId = dpid.zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700674 elif i == 28:
Jon Hallf37d44d2017-05-24 10:37:30 -0700675 deviceId = "2800".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -0700676 mappings[ macId ] = deviceId
677 if hosts[ controller ] or "Error" not in hosts[ controller ]:
678 if hosts[ controller ] == []:
679 main.log.warn( "There are no hosts discovered" )
680 zeroHosts = True
681 else:
682 for host in hosts[ controller ]:
683 mac = None
684 location = None
685 device = None
686 port = None
687 try:
688 mac = host.get( 'mac' )
689 assert mac, "mac field could not be found for this host object"
690
Jeremy Ronquillo0e538bc2017-06-13 15:16:09 -0700691 location = host.get( 'locations' )[ 0 ]
Jon Hall85794ff2015-07-08 14:12:30 -0700692 assert location, "location field could not be found for this host object"
693
694 # Trim the protocol identifier off deviceId
Jon Hallf37d44d2017-05-24 10:37:30 -0700695 device = str( location.get( 'elementId' ) ).split( ':' )[ 1 ]
Jon Hall85794ff2015-07-08 14:12:30 -0700696 assert device, "elementId field could not be found for this host location object"
697
698 port = location.get( 'port' )
699 assert port, "port field could not be found for this host location object"
700
701 # Now check if this matches where they should be
702 if mac and device and port:
703 if str( port ) != "1":
704 main.log.error( "The attachment port is incorrect for " +
705 "host " + str( mac ) +
Jon Hallf37d44d2017-05-24 10:37:30 -0700706 ". Expected: 1 Actual: " + str( port ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700707 hostAttachment = False
708 if device != mappings[ str( mac ) ]:
709 main.log.error( "The attachment device is incorrect for " +
710 "host " + str( mac ) +
711 ". Expected: " + mappings[ str( mac ) ] +
712 " Actual: " + device )
713 hostAttachment = False
714 else:
715 hostAttachment = False
716 except AssertionError:
717 main.log.exception( "Json object not as expected" )
718 main.log.error( repr( host ) )
719 hostAttachment = False
720 else:
721 main.log.error( "No hosts json output or \"Error\"" +
722 " in output. hosts = " +
723 repr( hosts[ controller ] ) )
724 if zeroHosts is False:
725 hostAttachment = True
726
Jon Hall85794ff2015-07-08 14:12:30 -0700727 devicesResults = devicesResults and currentDevicesResult
728 linksResults = linksResults and currentLinksResult
729 hostsResults = hostsResults and currentHostsResult
730 hostAttachmentResults = hostAttachmentResults and\
731 hostAttachment
732
Jon Halla440e872016-03-31 15:15:50 -0700733 # "consistent" results don't make sense for single instance
Jon Hall85794ff2015-07-08 14:12:30 -0700734 # there should always only be one cluster
Jon Hall85794ff2015-07-08 14:12:30 -0700735 clusterResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -0700736 try:
737 numClusters = len( json.loads( clusters[ 0 ] ) )
738 except ( ValueError, TypeError ):
739 main.log.exception( "Error parsing clusters[0]: " +
Jon Hallf37d44d2017-05-24 10:37:30 -0700740 repr( clusters[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -0700741 numClusters = "ERROR"
742 clusterResults = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700743 if numClusters == 1:
744 clusterResults = main.TRUE
745 utilities.assert_equals(
746 expect=1,
747 actual=numClusters,
748 onpass="ONOS shows 1 SCC",
749 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
750
751 topoResult = ( devicesResults and linksResults
752 and hostsResults and ipResult and clusterResults and
753 hostAttachmentResults )
754
755 topoResult = topoResult and int( count <= 2 )
756 note = "note it takes about " + str( int( cliTime ) ) + \
757 " seconds for the test to make all the cli calls to fetch " +\
758 "the topology from each ONOS instance"
759 main.log.info(
760 "Very crass estimate for topology discovery/convergence( " +
761 str( note ) + " ): " + str( elapsed ) + " seconds, " +
762 str( count ) + " tries" )
763 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
764 onpass="Topology Check Test successful",
765 onfail="Topology Check Test NOT successful" )
Jon Hall41d39f12016-04-11 22:54:35 -0700766 main.step( "Checking ONOS nodes" )
767 nodeResults = utilities.retry( main.HA.nodesCheck,
768 False,
Devin Lim142b5342017-07-20 15:22:39 -0700769 args=[ main.Cluster.active() ],
Jon Hall41d39f12016-04-11 22:54:35 -0700770 attempts=5 )
771
772 utilities.assert_equals( expect=True, actual=nodeResults,
773 onpass="Nodes check successful",
774 onfail="Nodes check NOT successful" )
775 if not nodeResults:
Devin Lim142b5342017-07-20 15:22:39 -0700776 for ctrl in main.Cluster.active():
Jon Hall41d39f12016-04-11 22:54:35 -0700777 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700778 ctrl.name,
779 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700780
Jon Halld2871c22016-07-26 11:01:14 -0700781 if not topoResult:
782 main.cleanup()
783 main.exit()
784
Jon Hall85794ff2015-07-08 14:12:30 -0700785 def CASE9( self, main ):
786 """
787 Link s3-s28 down
788 """
Devin Lim58046fa2017-07-05 16:55:00 -0700789 main.HA.linkDown( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700790
791 def CASE10( self, main ):
792 """
793 Link s3-s28 up
794 """
Devin Lim58046fa2017-07-05 16:55:00 -0700795 main.HA.linkUp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700796
797 def CASE11( self, main ):
798 """
799 Switch Down
800 """
801 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700802 main.HA.switchDown( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700803
804 def CASE12( self, main ):
805 """
806 Switch Up
807 """
808 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700809 main.HA.switchUp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700810
811 def CASE13( self, main ):
812 """
813 Clean up
814 """
Devin Lim58046fa2017-07-05 16:55:00 -0700815 main.HAlabels.append( "Restart" )
816 main.HAdata.append( str( main.restartTime ) )
817 main.HA.cleanUp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700818
819 def CASE14( self, main ):
820 """
821 start election app on all onos nodes
822 """
Devin Lim58046fa2017-07-05 16:55:00 -0700823 main.HA.startElectionApp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700824
825 def CASE15( self, main ):
826 """
827 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -0700828 15.1 Run election on each node
829 15.2 Check that each node has the same leaders and candidates
830 15.3 Find current leader and withdraw
831 15.4 Check that a new node was elected leader
832 15.5 Check that that new leader was the candidate of old leader
833 15.6 Run for election on old leader
834 15.7 Check that oldLeader is a candidate, and leader if only 1 node
835 15.8 Make sure that the old leader was added to the candidate list
836
837 old and new variable prefixes refer to data from before vs after
838 withdrawl and later before withdrawl vs after re-election
Jon Hall85794ff2015-07-08 14:12:30 -0700839 """
Devin Lim58046fa2017-07-05 16:55:00 -0700840 main.HA.isElectionFunctional( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700841
842 def CASE16( self, main ):
843 """
844 Install Distributed Primitives app
845 """
Devin Lim58046fa2017-07-05 16:55:00 -0700846 main.HA.installDistributedPrimitiveApp( main )
Jon Hall85794ff2015-07-08 14:12:30 -0700847
848 def CASE17( self, main ):
849 """
850 Check for basic functionality with distributed primitives
851 """
Devin Lim58046fa2017-07-05 16:55:00 -0700852 main.HA.checkDistPrimitivesFunc( main )