blob: 0aba15f7f62cbf6a12f3c4a6722ed5d715ad7188 [file] [log] [blame]
Jon Hall6e709752016-02-01 13:38:46 -08001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2016 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070021"""
Jon Hall6e709752016-02-01 13:38:46 -080022Description: This test is to determine if ONOS can handle
23 a full network partion
24
25List of test cases:
26CASE1: Compile ONOS and push it to the test machines
27CASE2: Assign devices to controllers
28CASE21: Assign mastership to controllers
29CASE3: Assign intents
30CASE4: Ping across added host intents
31CASE5: Reading state of ONOS
32CASE61: The Failure inducing case.
33CASE62: The Failure recovery case.
34CASE7: Check state after control plane failure
35CASE8: Compare topo
36CASE9: Link s3-s28 down
37CASE10: Link s3-s28 up
38CASE11: Switch down
39CASE12: Switch up
40CASE13: Clean up
41CASE14: start election app on all onos nodes
42CASE15: Check that Leadership Election is still functional
43CASE16: Install Distributed Primitives app
44CASE17: Check for basic functionality with distributed primitives
45"""
Jon Hall6e709752016-02-01 13:38:46 -080046class HAfullNetPartition:
47
48 def __init__( self ):
49 self.default = ''
50
51 def CASE1( self, main ):
52 """
53 CASE1 is to compile ONOS and push it to the test machines
54
55 Startup sequence:
56 cell <name>
57 onos-verify-cell
58 NOTE: temporary - onos-remove-raft-logs
59 onos-uninstall
60 start mininet
61 git pull
62 mvn clean install
63 onos-package
64 onos-install -f
65 onos-wait-for-start
66 start cli sessions
67 start tcpdump
68 """
Jon Hall6e709752016-02-01 13:38:46 -080069 main.log.info( "ONOS HA test: Partition ONOS nodes into two sub-clusters - " +
70 "initialization" )
Jon Hall6e709752016-02-01 13:38:46 -080071 # set global variables
Jon Halla440e872016-03-31 15:15:50 -070072 # These are for csv plotting in jenkins
Devin Lim58046fa2017-07-05 16:55:00 -070073 main.HAlabels = []
74 main.HAdata = []
75 try:
76 from tests.dependencies.ONOSSetup import ONOSSetup
77 main.testSetUp = ONOSSetup()
78 except ImportError:
79 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070080 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070081 main.testSetUp.envSetupDescription()
Jon Hall6e709752016-02-01 13:38:46 -080082 try:
Jon Hall53c5e662016-04-13 16:06:56 -070083 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070084 main.HA = HA()
Devin Lim58046fa2017-07-05 16:55:00 -070085 # load some variables from the params file
86 cellName = main.params[ 'ENV' ][ 'cellName' ]
87 main.apps = main.params[ 'ENV' ][ 'appString' ]
Jon Hallab611372018-02-21 15:26:05 -080088 stepResult = main.testSetUp.envSetup( includeCaseDesc=False )
Jon Hall6e709752016-02-01 13:38:46 -080089 except Exception as e:
Devin Lim58046fa2017-07-05 16:55:00 -070090 main.testSetUp.envSetupException( e )
91 main.testSetUp.evnSetupConclusion( stepResult )
Jon Hall6e709752016-02-01 13:38:46 -080092
Jon Hall5a5c8432018-11-28 11:39:57 -080093 applyFuncs = [ main.HA.removeKarafConsoleLogging, main.HA.customizeOnosGenPartitions ]
94 applyArgs = [ None, None ]
Jon Hallab611372018-02-21 15:26:05 -080095 try:
96 if main.params[ 'topology' ][ 'topoFile' ]:
97 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
98 else:
99 applyFuncs.append( main.HA.startingMininet )
100 applyArgs.append( None )
101 except (KeyError, IndexError):
102 applyFuncs.append( main.HA.startingMininet )
103 applyArgs.append( None )
104
Jon Hall3e6edb32018-08-21 16:20:30 -0700105 main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
Jon Hallab611372018-02-21 15:26:05 -0800106 extraApply=applyFuncs,
107 applyArgs=applyArgs,
108 extraClean=main.HA.cleanUpGenPartition,
109 includeCaseDesc=False )
Devin Lim58046fa2017-07-05 16:55:00 -0700110 main.HA.initialSetUp()
Jon Hall6e709752016-02-01 13:38:46 -0800111
Jon Hallab611372018-02-21 15:26:05 -0800112 main.step( 'Set logging levels' )
113 logging = True
114 try:
115 logs = main.params.get( 'ONOS_Logging', False )
116 if logs:
117 for namespace, level in logs.items():
118 for ctrl in main.Cluster.active():
119 ctrl.CLI.logSet( level, namespace )
120 except AttributeError:
121 logging = False
122 utilities.assert_equals( expect=True, actual=logging,
123 onpass="Set log levels",
124 onfail="Failed to set log levels" )
125
Jon Hall6e709752016-02-01 13:38:46 -0800126 def CASE2( self, main ):
127 """
128 Assign devices to controllers
129 """
Devin Lim58046fa2017-07-05 16:55:00 -0700130 main.HA.assignDevices( main )
Jon Hall6e709752016-02-01 13:38:46 -0800131
Jon Hallab611372018-02-21 15:26:05 -0800132 def CASE102( self, main ):
133 """
134 Set up Spine-Leaf fabric topology in Mininet
135 """
136 main.HA.startTopology( main )
137
Jon Hall6e709752016-02-01 13:38:46 -0800138 def CASE21( self, main ):
139 """
140 Assign mastership to controllers
141 """
Devin Lim58046fa2017-07-05 16:55:00 -0700142 main.HA.assignMastership( main )
Jon Hall6e709752016-02-01 13:38:46 -0800143
144 def CASE3( self, main ):
145 """
146 Assign intents
147 """
Devin Lim58046fa2017-07-05 16:55:00 -0700148 main.HA.assignIntents( main )
Jon Hall6e709752016-02-01 13:38:46 -0800149
150 def CASE4( self, main ):
151 """
152 Ping across added host intents
153 """
Jon Hallca319892017-06-15 15:25:22 -0700154 main.HA.pingAcrossHostIntent( main )
Jon Hall6e709752016-02-01 13:38:46 -0800155
Jon Hallab611372018-02-21 15:26:05 -0800156 def CASE104( self, main ):
157 """
158 Ping Hosts
159 """
160 main.case( "Check connectivity" )
161 main.step( "Ping between all hosts" )
162 pingResult = main.Mininet1.pingall()
163 utilities.assert_equals( expect=main.TRUE, actual=pingResult,
164 onpass="All Pings Passed",
165 onfail="Failed to ping between all hosts" )
166
Jon Hall6e709752016-02-01 13:38:46 -0800167 def CASE5( self, main ):
168 """
169 Reading state of ONOS
170 """
Devin Lim58046fa2017-07-05 16:55:00 -0700171 main.HA.readingState( main )
Jon Hall6e709752016-02-01 13:38:46 -0800172
173 def CASE61( self, main ):
174 """
175 The Failure case.
176 """
Jon Halla478b852017-12-04 15:00:15 -0800177 import pexpect
178 import time
Jon Hall6e709752016-02-01 13:38:46 -0800179 assert main, "main not defined"
180 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Hall6e709752016-02-01 13:38:46 -0800181 main.case( "Partition ONOS nodes into two distinct partitions" )
182
183 main.step( "Checking ONOS Logs for errors" )
Devin Lim142b5342017-07-20 15:22:39 -0700184 for ctrl in main.Cluster.runningNodes:
185 main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
186 main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
Jon Hall6e709752016-02-01 13:38:46 -0800187
Devin Lim142b5342017-07-20 15:22:39 -0700188 main.log.debug( main.Cluster.next().CLI.roles( jsonFormat=False ) )
Jon Halld2871c22016-07-26 11:01:14 -0700189
Devin Lim142b5342017-07-20 15:22:39 -0700190 n = len( main.Cluster.runningNodes ) # Number of nodes
Jon Hall6e709752016-02-01 13:38:46 -0800191 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
192 main.partition = [ 0 ] # ONOS node to partition, listed by index in main.nodes
193 if n > 3:
194 main.partition.append( p - 1 )
195 # NOTE: This only works for cluster sizes of 3,5, or 7.
196
197 main.step( "Partitioning ONOS nodes" )
198 nodeList = [ str( i + 1 ) for i in main.partition ]
199 main.log.info( "Nodes to be partitioned: " + str( nodeList ) )
200 partitionResults = main.TRUE
Jon Hall701fea12018-10-08 11:09:22 -0700201 standalonePort = "5679" # use params file and default to this
202 embeddedPort = "9876" # use params file and default to this
203 # We are blocking traffic from the embedded Atomix instance on nodes in main.partition
Jon Hall6e709752016-02-01 13:38:46 -0800204 for i in range( 0, n ):
Devin Lim142b5342017-07-20 15:22:39 -0700205 iCtrl = main.Cluster.runningNodes[ i ]
206 this = iCtrl.Bench.sshToNode( iCtrl.ipAddress )
Jon Hall701fea12018-10-08 11:09:22 -0700207 if i not in main.partition: # i is in the majority partition
208 for j in main.partition: # j is in the minority partition
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700209 foe = main.Cluster.runningNodes[ j ]
Jon Halla478b852017-12-04 15:00:15 -0800210 main.log.warn( "Setting IP Tables rule from {} to {}. ".format( iCtrl.ipAddress,
211 foe.ipAddress ) )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700212 # CMD HERE
Jon Hall701fea12018-10-08 11:09:22 -0700213 # On node i in majority partition: add rule to block from j in minority partition
214 cmdList = [ ]
215 cmdList.append( "sudo iptables -A {} -p tcp --sp {} -d {} -s {} -j DROP".format( "INPUT",
216 embeddedPort,
217 iCtrl.ipAddress,
218 foe.ipAddress ) )
219 cmdList.append( "sudo ip6tables -A {} -p tcp --sp {} -d {} -s {} -j DROP".format( "INPUT",
220 embeddedPort,
221 iCtrl.ipAddress,
222 foe.ipAddress ) )
Devin Lim142b5342017-07-20 15:22:39 -0700223 try:
Jon Hall701fea12018-10-08 11:09:22 -0700224 for cmd in cmdList:
225 this.sendline( cmd )
226 this.expect( "\$" )
227 main.log.debug( this.before )
Devin Lim142b5342017-07-20 15:22:39 -0700228 except pexpect.EOF:
Jon Halla478b852017-12-04 15:00:15 -0800229 main.log.error( iCtrl.name + ": EOF exception found" )
230 main.log.error( iCtrl.name + ": " + this.before )
Devin Lim44075962017-08-11 10:56:37 -0700231 main.cleanAndExit()
Devin Lim142b5342017-07-20 15:22:39 -0700232 except Exception:
Jon Halla478b852017-12-04 15:00:15 -0800233 main.log.exception( iCtrl.name + ": Uncaught exception!" )
Devin Lim44075962017-08-11 10:56:37 -0700234 main.cleanAndExit()
Jon Hall6e709752016-02-01 13:38:46 -0800235 else:
236 for j in range( 0, n ):
237 if j not in main.partition:
Jon Hall701fea12018-10-08 11:09:22 -0700238 # On node i in minority partition: add rule to block from j in majority partition
Devin Lim142b5342017-07-20 15:22:39 -0700239 foe = main.Cluster.runningNodes[ j ]
Jon Halla478b852017-12-04 15:00:15 -0800240 main.log.warn( "Setting IP Tables rule from {} to {}. ".format( iCtrl.ipAddress,
241 foe.ipAddress ) )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700242 # CMD HERE
Jon Hall701fea12018-10-08 11:09:22 -0700243 cmdList = [ ]
244 cmdList.append( "sudo iptables -A {} -p tcp --dp {} -d {} -s {} -j DROP".format( "INPUT",
245 embeddedPort,
246 iCtrl.ipAddress,
247 foe.ipAddress ) )
248 cmdList.append( "sudo ip6tables -A {} -p tcp --dp {} -d {} -s {} -j DROP".format( "INPUT",
249 embeddedPort,
250 iCtrl.ipAddress,
251 foe.ipAddress ) )
252 cmdList.append( "sudo iptables -A {} -p tcp --sp {} -s {} -j DROP".format( "OUTPUT",
253 embeddedPort,
254 iCtrl.ipAddress ) )
255 cmdList.append( "sudo ip6tables -A {} -p tcp --sp {} -s {} -j DROP".format( "OUTPUT",
256 embeddedPort,
257 iCtrl.ipAddress ) )
258 cmdList.append( "sudo iptables -A {} -p tcp --sp {} -s {} -m conntrack --ctstate ESTABLISHED -j DROP".format( "OUTPUT",
259 embeddedPort,
260 iCtrl.ipAddress ) )
261 cmdList.append( "sudo ip6tables -A {} -p tcp --sp {} -s {} -m conntrack --ctstate ESTABLISHED -j DROP".format( "OUTPUT",
262 embeddedPort,
263 iCtrl.ipAddress ) )
264 cmdList.append( "sudo iptables -A {} -p tcp --sp {} -s {} -m conntrack --ctstate ESTABLISHED -j DROP".format( "INPUT",
265 embeddedPort,
266 foe.ipAddress ) )
267 cmdList.append( "sudo ip6tables -A {} -p tcp --sp {} -s {} -m conntrack --ctstate ESTABLISHED -j DROP".format( "INPUT",
268 embeddedPort,
269 foe.ipAddress ) )
Devin Lim142b5342017-07-20 15:22:39 -0700270 try:
Jon Hall701fea12018-10-08 11:09:22 -0700271 for cmd in cmdList:
272 this.sendline( cmd )
273 this.expect( "\$" )
274 main.log.debug( this.before )
Devin Lim142b5342017-07-20 15:22:39 -0700275 except pexpect.EOF:
Jon Halla478b852017-12-04 15:00:15 -0800276 main.log.error( iCtrl.name + ": EOF exception found" )
277 main.log.error( iCtrl.name + ": " + this.before )
Devin Lim44075962017-08-11 10:56:37 -0700278 main.cleanAndExit()
Devin Lim142b5342017-07-20 15:22:39 -0700279 except Exception:
Jon Halla478b852017-12-04 15:00:15 -0800280 main.log.exception( iCtrl.name + ": Uncaught exception!" )
Devin Lim44075962017-08-11 10:56:37 -0700281 main.cleanAndExit()
Jon Hall701fea12018-10-08 11:09:22 -0700282 # From embedded atomix to standalone atomix on same node, node i in minority partition
283 foe = main.Cluster.runningNodes[ i ]
284 main.log.warn( "Setting IP Tables rule from {} to {}. ".format( iCtrl.ipAddress,
285 foe.ipAddress ) )
286 # CMD HERE
287 cmdStr = "sudo iptables -A {} -p tcp --sp {} -d {} -s {} -j DROP".format( "INPUT",
288 embeddedPort,
289 iCtrl.ipAddress,
290 foe.ipAddress )
291 cmdStr2 = "sudo iptables -A {} -p tcp --dp {} -d {} -s {} -j DROP".format( "INPUT",
292 embeddedPort,
293 iCtrl.ipAddress,
294 foe.ipAddress )
295 cmdStr3 = "sudo iptables -A {} -p tcp --sp {} -j DROP".format( "OUTPUT",
296 embeddedPort )
297 try:
298 this.sendline( cmdStr )
299 this.expect( "\$" )
300 main.log.debug( this.before )
301 this.sendline( cmdStr2 )
302 this.expect( "\$" )
303 main.log.debug( this.before )
304 except pexpect.EOF:
305 main.log.error( iCtrl.name + ": EOF exception found" )
306 main.log.error( iCtrl.name + ": " + this.before )
307 main.cleanAndExit()
308 except Exception:
309 main.log.exception( iCtrl.name + ": Uncaught exception!" )
310 main.cleanAndExit()
Devin Lim142b5342017-07-20 15:22:39 -0700311 main.Cluster.runningNodes[ i ].active = False
Jon Hall701fea12018-10-08 11:09:22 -0700312 # end if/else
Devin Lim142b5342017-07-20 15:22:39 -0700313 iCtrl.Bench.exitFromSsh( this, iCtrl.ipAddress )
Jon Hall6e709752016-02-01 13:38:46 -0800314 utilities.assert_equals( expect=main.TRUE, actual=partitionResults,
315 onpass="Firewall rules set successfully",
316 onfail="Error setting firewall rules" )
Jon Halla1e8e512018-05-11 13:30:57 -0700317 main.Cluster.reset()
Jon Hall701fea12018-10-08 11:09:22 -0700318 main.log.debug( main.Cluster.active )
Jon Hall6e709752016-02-01 13:38:46 -0800319
Jon Hall6509dbf2016-06-21 17:01:17 -0700320 main.step( "Sleeping 60 seconds" )
Jon Hall6e709752016-02-01 13:38:46 -0800321 time.sleep( 60 )
322
323 def CASE62( self, main ):
324 """
325 Healing Partition
326 """
Jon Hall6e709752016-02-01 13:38:46 -0800327 assert main, "main not defined"
328 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Hall6e709752016-02-01 13:38:46 -0800329 assert main.partition, "main.partition not defined"
330 main.case( "Healing Partition" )
331
332 main.step( "Deleteing firewall rules" )
333 healResults = main.TRUE
Devin Lim142b5342017-07-20 15:22:39 -0700334 for ctrl in main.Cluster.runningNodes:
Jon Hall6e709752016-02-01 13:38:46 -0800335 cmdStr = "sudo iptables -F"
Devin Lim142b5342017-07-20 15:22:39 -0700336 handle = ctrl.Bench.sshToNode( ctrl.ipAddress )
337 handle.sendline( cmdStr )
338 handle.expect( "\$" )
339 main.log.debug( handle.before )
340 ctrl.Bench.exitFromSsh( handle, ctrl.ipAddress )
Jon Hall6e709752016-02-01 13:38:46 -0800341 utilities.assert_equals( expect=main.TRUE, actual=healResults,
342 onpass="Firewall rules removed",
343 onfail="Error removing firewall rules" )
344
345 for node in main.partition:
Devin Lim142b5342017-07-20 15:22:39 -0700346 main.Cluster.runningNodes[ node ].active = True
Jon Halla1e8e512018-05-11 13:30:57 -0700347 main.Cluster.reset()
Devin Lim142b5342017-07-20 15:22:39 -0700348
Jon Hall30668ff2019-02-27 17:43:09 -0800349 main.testSetUp.checkOnosNodes( main.Cluster )
Jon Halld2871c22016-07-26 11:01:14 -0700350
Jon Hall6e709752016-02-01 13:38:46 -0800351 def CASE7( self, main ):
352 """
353 Check state after ONOS failure
354 """
Devin Lim142b5342017-07-20 15:22:39 -0700355 main.HA.checkStateAfterEvent( main, afterWhich=0 )
Jon Hall6e709752016-02-01 13:38:46 -0800356
Jon Hall6e709752016-02-01 13:38:46 -0800357 main.step( "Leadership Election is still functional" )
358 # Test of LeadershipElection
359 leaderList = []
360
361 partitioned = []
362 for i in main.partition:
Devin Lim142b5342017-07-20 15:22:39 -0700363 partitioned.append( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall6e709752016-02-01 13:38:46 -0800364 leaderResult = main.TRUE
365
Devin Lim142b5342017-07-20 15:22:39 -0700366 for ctrl in main.Cluster.active():
367 leaderN = ctrl.CLI.electionTestLeader()
Jon Hall6e709752016-02-01 13:38:46 -0800368 leaderList.append( leaderN )
369 if leaderN == main.FALSE:
370 # error in response
371 main.log.error( "Something is wrong with " +
372 "electionTestLeader function, check the" +
373 " error logs" )
374 leaderResult = main.FALSE
375 elif leaderN is None:
Devin Lim142b5342017-07-20 15:22:39 -0700376 main.log.error( ctrl.name +
Jon Hall6e709752016-02-01 13:38:46 -0800377 " shows no leader for the election-app was" +
378 " elected after the old one died" )
379 leaderResult = main.FALSE
380 elif leaderN in partitioned:
Devin Lim142b5342017-07-20 15:22:39 -0700381 main.log.error( ctrl.name + " shows " + str( leaderN ) +
Jon Hall6e709752016-02-01 13:38:46 -0800382 " as leader for the election-app, but it " +
383 "was partitioned" )
384 leaderResult = main.FALSE
385 if len( set( leaderList ) ) != 1:
386 leaderResult = main.FALSE
387 main.log.error(
388 "Inconsistent view of leader for the election test app" )
Jon Hallab611372018-02-21 15:26:05 -0800389 main.log.debug( leaderList )
Jon Hall6e709752016-02-01 13:38:46 -0800390 utilities.assert_equals(
391 expect=main.TRUE,
392 actual=leaderResult,
393 onpass="Leadership election passed",
394 onfail="Something went wrong with Leadership election" )
395
396 def CASE8( self, main ):
397 """
398 Compare topo
399 """
Devin Lim58046fa2017-07-05 16:55:00 -0700400 main.HA.compareTopo( main )
Jon Halld2871c22016-07-26 11:01:14 -0700401
Jon Hall6e709752016-02-01 13:38:46 -0800402 def CASE9( self, main ):
403 """
Jon Hallab611372018-02-21 15:26:05 -0800404 Link down
Jon Hall6e709752016-02-01 13:38:46 -0800405 """
Jon Hallab611372018-02-21 15:26:05 -0800406 src = main.params['kill']['linkSrc']
407 dst = main.params['kill']['linkDst']
408 main.HA.linkDown( main, src, dst )
Jon Hall6e709752016-02-01 13:38:46 -0800409
410 def CASE10( self, main ):
411 """
Jon Hallab611372018-02-21 15:26:05 -0800412 Link up
Jon Hall6e709752016-02-01 13:38:46 -0800413 """
Jon Hallab611372018-02-21 15:26:05 -0800414 src = main.params['kill']['linkSrc']
415 dst = main.params['kill']['linkDst']
416 main.HA.linkUp( main, src, dst )
Jon Hall6e709752016-02-01 13:38:46 -0800417
418 def CASE11( self, main ):
419 """
420 Switch Down
421 """
422 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700423 main.HA.switchDown( main )
Jon Hall6e709752016-02-01 13:38:46 -0800424
425 def CASE12( self, main ):
426 """
427 Switch Up
428 """
429 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700430 main.HA.switchUp( main )
Jon Hall6e709752016-02-01 13:38:46 -0800431
432 def CASE13( self, main ):
433 """
434 Clean up
435 """
Devin Lim58046fa2017-07-05 16:55:00 -0700436 main.HA.cleanUp( main )
Jon Hall6e709752016-02-01 13:38:46 -0800437
438 def CASE14( self, main ):
439 """
Jon Hallab611372018-02-21 15:26:05 -0800440 Start election app on all onos nodes
Jon Hall6e709752016-02-01 13:38:46 -0800441 """
Devin Lim58046fa2017-07-05 16:55:00 -0700442 main.HA.startElectionApp( main )
Jon Hall6e709752016-02-01 13:38:46 -0800443
444 def CASE15( self, main ):
445 """
446 Check that Leadership Election is still functional
447 15.1 Run election on each node
448 15.2 Check that each node has the same leaders and candidates
449 15.3 Find current leader and withdraw
450 15.4 Check that a new node was elected leader
451 15.5 Check that that new leader was the candidate of old leader
452 15.6 Run for election on old leader
453 15.7 Check that oldLeader is a candidate, and leader if only 1 node
454 15.8 Make sure that the old leader was added to the candidate list
455
456 old and new variable prefixes refer to data from before vs after
457 withdrawl and later before withdrawl vs after re-election
458 """
Devin Lim58046fa2017-07-05 16:55:00 -0700459 main.HA.isElectionFunctional( main )
Jon Hall6e709752016-02-01 13:38:46 -0800460
461 def CASE16( self, main ):
462 """
463 Install Distributed Primitives app
464 """
Devin Lim58046fa2017-07-05 16:55:00 -0700465 main.HA.installDistributedPrimitiveApp( main )
Jon Hall6e709752016-02-01 13:38:46 -0800466
467 def CASE17( self, main ):
468 """
469 Check for basic functionality with distributed primitives
470 """
Devin Lim58046fa2017-07-05 16:55:00 -0700471 main.HA.checkDistPrimitivesFunc( main )