blob: 923f67c3e47821fb4be53fe3f7189522074e73fd [file] [log] [blame]
Jon Hall69b2b982016-05-11 12:04:59 -07001"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07002Copyright 2016 Open Networking Foundation (ONF)
3
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
11 (at your option) any later version.
12
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
21
22"""
Jon Hall69b2b982016-05-11 12:04:59 -070023Description: This test is to determine if ONOS can handle
24 dynamic swapping of cluster nodes.
25
26List of test cases:
27CASE1: Compile ONOS and push it to the test machines
28CASE2: Assign devices to controllers
29CASE21: Assign mastership to controllers
30CASE3: Assign intents
31CASE4: Ping across added host intents
32CASE5: Reading state of ONOS
33CASE6: Swap nodes
34CASE7: Check state after control plane failure
35CASE8: Compare topo
36CASE9: Link s3-s28 down
37CASE10: Link s3-s28 up
38CASE11: Switch down
39CASE12: Switch up
40CASE13: Clean up
41CASE14: start election app on all onos nodes
42CASE15: Check that Leadership Election is still functional
43CASE16: Install Distributed Primitives app
44CASE17: Check for basic functionality with distributed primitives
45"""
Jon Hall69b2b982016-05-11 12:04:59 -070046class HAswapNodes:
47
48 def __init__( self ):
49 self.default = ''
50
51 def CASE1( self, main ):
52 """
53 CASE1 is to compile ONOS and push it to the test machines
54
55 Startup sequence:
56 cell <name>
57 onos-verify-cell
58 NOTE: temporary - onos-remove-raft-logs
59 onos-uninstall
60 start mininet
61 git pull
62 mvn clean install
63 onos-package
64 onos-install -f
65 onos-wait-for-start
66 start cli sessions
67 start tcpdump
68 """
69 import time
70 import os
71 import re
72 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
73 "initialization" )
Jon Hall69b2b982016-05-11 12:04:59 -070074 # set global variables
75 # These are for csv plotting in jenkins
Devin Lim58046fa2017-07-05 16:55:00 -070076 main.HAlabels = []
77 main.HAdata = []
78 try:
79 from tests.dependencies.ONOSSetup import ONOSSetup
80 main.testSetUp = ONOSSetup()
81 except ImportError:
82 main.log.error( "ONOSSetup not found. exiting the test" )
83 main.exit()
84 main.testSetUp.envSetupDescription()
Jon Hall69b2b982016-05-11 12:04:59 -070085 try:
86 from tests.HA.dependencies.HA import HA
87 main.HA = HA()
88 from tests.HA.HAswapNodes.dependencies.Server import Server
89 main.Server = Server()
Devin Lim58046fa2017-07-05 16:55:00 -070090 # load some variables from the params file
91 cellName = main.params[ 'ENV' ][ 'cellName' ]
92 main.apps = main.params[ 'ENV' ][ 'appString' ]
Devin Lim142b5342017-07-20 15:22:39 -070093 stepResult = main.testSetUp.envSetup()
Jon Hall69b2b982016-05-11 12:04:59 -070094 except Exception as e:
Devin Lim58046fa2017-07-05 16:55:00 -070095 main.testSetUp.envSetupException( e )
96 main.testSetUp.evnSetupConclusion( stepResult )
97 main.HA.generateGraph( "HAswapNodes" )
Jon Hall69b2b982016-05-11 12:04:59 -070098
Jon Hall69b2b982016-05-11 12:04:59 -070099
Devin Lim142b5342017-07-20 15:22:39 -0700100 main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName=cellName, removeLog=True,
101 extraApply=[ main.HA.setServerForCluster,
102 main.HA.swapNodeMetadata,
103 main.HA.startingMininet,
104 main.HA.copyingBackupConfig ],
Devin Lim58046fa2017-07-05 16:55:00 -0700105 extraClean=main.HA.cleanUpOnosService,
106 installMax=True )
107 main.HA.initialSetUp()
Jon Hall69b2b982016-05-11 12:04:59 -0700108
109 def CASE2( self, main ):
110 """
111 Assign devices to controllers
112 """
Devin Lim58046fa2017-07-05 16:55:00 -0700113 main.HA.assignDevices( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700114
115 def CASE21( self, main ):
116 """
117 Assign mastership to controllers
118 """
Devin Lim58046fa2017-07-05 16:55:00 -0700119 main.HA.assignMastership( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700120
121 def CASE3( self, main ):
122 """
123 Assign intents
124 """
Devin Lim58046fa2017-07-05 16:55:00 -0700125 main.HA.assignIntents( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700126
127 def CASE4( self, main ):
128 """
129 Ping across added host intents
130 """
Jon Hallca319892017-06-15 15:25:22 -0700131 main.HA.pingAcrossHostIntent( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700132
133 def CASE5( self, main ):
134 """
135 Reading state of ONOS
136 """
Devin Lim58046fa2017-07-05 16:55:00 -0700137 main.HA.readingState( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700138
139 def CASE6( self, main ):
140 """
141 The Scaling case.
142 """
143 import time
144 import re
Jon Hall69b2b982016-05-11 12:04:59 -0700145 assert main, "main not defined"
146 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Hall69b2b982016-05-11 12:04:59 -0700147 try:
Devin Lim58046fa2017-07-05 16:55:00 -0700148 main.HAlabels
149 except ( NameError, AttributeError ):
150 main.log.error( "main.HAlabels not defined, setting to []" )
151 main.HAlabels = []
Jon Hall69b2b982016-05-11 12:04:59 -0700152 try:
Devin Lim58046fa2017-07-05 16:55:00 -0700153 main.HAdata
154 except ( NameError, AttributeError ):
155 main.log.error( "main.HAdata not defined, setting to []" )
156 main.HAdata = []
Jon Hall69b2b982016-05-11 12:04:59 -0700157
158 main.case( "Swap some of the ONOS nodes" )
159
160 main.step( "Checking ONOS Logs for errors" )
Devin Lim142b5342017-07-20 15:22:39 -0700161 for ctrl in main.Cluster.active():
162 main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
163 main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
Jon Hall69b2b982016-05-11 12:04:59 -0700164
Devin Lim142b5342017-07-20 15:22:39 -0700165 activeNodes = main.Cluster.getRunningPos()
166 # Todo : this could be wrong. need to double check.
Jon Hall69b2b982016-05-11 12:04:59 -0700167 main.step( "Generate new metadata file" )
Devin Lim142b5342017-07-20 15:22:39 -0700168 old = [ activeNodes[ 1 ], activeNodes[ -2 ] ]
169 new = range( main.Cluster.maxCtrls )[ -2: ]
Jon Hall69b2b982016-05-11 12:04:59 -0700170 assert len( old ) == len( new ), "Length of nodes to swap don't match"
171 handle = main.ONOSbench.handle
172 for x, y in zip( old, new ):
173 handle.sendline( "export OC{}=$OC{}".format( x + 1, y + 1 ) )
174 handle.expect( "\$" ) # from the variable
175 ret = handle.before
176 handle.expect( "\$" ) # From the prompt
177 ret += handle.before
178 main.log.debug( ret )
Devin Lim142b5342017-07-20 15:22:39 -0700179 activeNodes.remove( x )
180 activeNodes.append( y )
Jon Hall69b2b982016-05-11 12:04:59 -0700181
Devin Lim142b5342017-07-20 15:22:39 -0700182 genResult = main.Server.generateFile( main.Cluster.numCtrls )
Jon Hall69b2b982016-05-11 12:04:59 -0700183 utilities.assert_equals( expect=main.TRUE, actual=genResult,
184 onpass="New cluster metadata file generated",
185 onfail="Failled to generate new metadata file" )
186 time.sleep( 5 ) # Give time for nodes to read new file
Devin Lim142b5342017-07-20 15:22:39 -0700187 main.Cluster.resetActive()
188 # Note : done up to this point.
Jon Hall69b2b982016-05-11 12:04:59 -0700189 main.step( "Start new nodes" ) # OR stop old nodes?
190 started = main.TRUE
191 for i in new:
Devin Lim142b5342017-07-20 15:22:39 -0700192 started = main.ONOSbench.onosStart( main.Cluster.controllers[ i ].ipAddress ) and main.TRUE
Jon Hall69b2b982016-05-11 12:04:59 -0700193 utilities.assert_equals( expect=main.TRUE, actual=started,
194 onpass="ONOS started",
195 onfail="ONOS start NOT successful" )
196
Devin Lim142b5342017-07-20 15:22:39 -0700197 main.Cluster.setRunningNode( activeNodes )
Jon Hall69b2b982016-05-11 12:04:59 -0700198
Devin Lim142b5342017-07-20 15:22:39 -0700199 main.testSetUp.setupSsh( main.Cluster )
200 main.testSetUp.checkOnosService( main.Cluster )
Jon Hall69b2b982016-05-11 12:04:59 -0700201
Devin Lim142b5342017-07-20 15:22:39 -0700202 main.testSetUp.startOnosClis( main.Cluster )
Jon Hall69b2b982016-05-11 12:04:59 -0700203
204 main.step( "Checking ONOS nodes" )
205 nodeResults = utilities.retry( main.HA.nodesCheck,
206 False,
Devin Lim142b5342017-07-20 15:22:39 -0700207 args=[ main.Cluster.active() ],
Jon Hall69b2b982016-05-11 12:04:59 -0700208 attempts=5 )
209 utilities.assert_equals( expect=True, actual=nodeResults,
210 onpass="Nodes check successful",
211 onfail="Nodes check NOT successful" )
212
Devin Lim142b5342017-07-20 15:22:39 -0700213 ready = utilities.retry( main.Cluster.command,
214 False,
215 kwargs={ "function":"summary", "contentCheck":True },
216 sleep=30,
217 attempts=10 )
Jon Hall69b2b982016-05-11 12:04:59 -0700218 utilities.assert_equals( expect=True, actual=ready,
219 onpass="ONOS summary command succeded",
220 onfail="ONOS summary command failed" )
221 if not ready:
222 main.cleanup()
223 main.exit()
224
225 # Rerun for election on new nodes
226 runResults = main.TRUE
Devin Lim142b5342017-07-20 15:22:39 -0700227 for ctrl in main.Cluster.active():
228 run = ctrl.CLI.electionTestRun()
Jon Hall69b2b982016-05-11 12:04:59 -0700229 if run != main.TRUE:
Devin Lim142b5342017-07-20 15:22:39 -0700230 main.log.error( "Error running for election on " + ctrl.name )
Jon Hall69b2b982016-05-11 12:04:59 -0700231 runResults = runResults and run
232 utilities.assert_equals( expect=main.TRUE, actual=runResults,
233 onpass="Reran for election",
234 onfail="Failed to rerun for election" )
235
Devin Lim142b5342017-07-20 15:22:39 -0700236 main.HA.commonChecks()
237
238 """
239 # Note: Do we really want this? It will revert the changes we have made from this test case.
Jon Hall69b2b982016-05-11 12:04:59 -0700240
241 main.step( "Reapplying cell variable to environment" )
242 cellName = main.params[ 'ENV' ][ 'cellName' ]
243 cellResult = main.ONOSbench.setCell( cellName )
244 utilities.assert_equals( expect=main.TRUE, actual=cellResult,
245 onpass="Set cell successfull",
246 onfail="Failled to set cell" )
Devin Lim142b5342017-07-20 15:22:39 -0700247 """
248
Jon Hall69b2b982016-05-11 12:04:59 -0700249
250 def CASE7( self, main ):
251 """
252 Check state after ONOS scaling
253 """
Jon Hall69b2b982016-05-11 12:04:59 -0700254
Devin Lim142b5342017-07-20 15:22:39 -0700255 main.HA.checkStateAfterEvent( main, afterWhich=1 )
Jon Hall69b2b982016-05-11 12:04:59 -0700256
Jon Hall69b2b982016-05-11 12:04:59 -0700257 main.step( "Leadership Election is still functional" )
258 # Test of LeadershipElection
259 leaderList = []
260 leaderResult = main.TRUE
261
Devin Lim142b5342017-07-20 15:22:39 -0700262 for ctrl in main.Cluster.active():
263 leaderN = ctrl.CLI.electionTestLeader()
Jon Hall69b2b982016-05-11 12:04:59 -0700264 leaderList.append( leaderN )
265 if leaderN == main.FALSE:
266 # error in response
267 main.log.error( "Something is wrong with " +
268 "electionTestLeader function, check the" +
269 " error logs" )
270 leaderResult = main.FALSE
271 elif leaderN is None:
Devin Lim142b5342017-07-20 15:22:39 -0700272 main.log.error( ctrl.name +
Jon Hall69b2b982016-05-11 12:04:59 -0700273 " shows no leader for the election-app." )
274 leaderResult = main.FALSE
275 if len( set( leaderList ) ) != 1:
276 leaderResult = main.FALSE
277 main.log.error(
278 "Inconsistent view of leader for the election test app" )
279 # TODO: print the list
280 utilities.assert_equals(
281 expect=main.TRUE,
282 actual=leaderResult,
283 onpass="Leadership election passed",
284 onfail="Something went wrong with Leadership election" )
285
286 def CASE8( self, main ):
287 """
288 Compare topo
289 """
Devin Lim58046fa2017-07-05 16:55:00 -0700290 main.HA.compareTopo( main )
Jon Halld2871c22016-07-26 11:01:14 -0700291
Jon Hall69b2b982016-05-11 12:04:59 -0700292 def CASE9( self, main ):
293 """
294 Link s3-s28 down
295 """
Devin Lim58046fa2017-07-05 16:55:00 -0700296 main.HA.linkDown( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700297
298 def CASE10( self, main ):
299 """
300 Link s3-s28 up
301 """
Devin Lim58046fa2017-07-05 16:55:00 -0700302 main.HA.linkUp( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700303
304 def CASE11( self, main ):
305 """
306 Switch Down
307 """
308 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700309 main.HA.switchDown( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700310
311 def CASE12( self, main ):
312 """
313 Switch Up
314 """
315 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700316 main.HA.switchUp( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700317
318 def CASE13( self, main ):
319 """
320 Clean up
321 """
Devin Lim58046fa2017-07-05 16:55:00 -0700322 main.HA.cleanUp( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700323
324 main.step( "Stopping webserver" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700325 status = main.Server.stop()
Jon Hall69b2b982016-05-11 12:04:59 -0700326 utilities.assert_equals( expect=main.TRUE, actual=status,
327 onpass="Stop Server",
328 onfail="Failled to stop SimpleHTTPServer" )
329 del main.Server
330
331 def CASE14( self, main ):
332 """
333 start election app on all onos nodes
334 """
Devin Lim58046fa2017-07-05 16:55:00 -0700335 main.HA.startElectionApp( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700336
337 def CASE15( self, main ):
338 """
339 Check that Leadership Election is still functional
340 15.1 Run election on each node
341 15.2 Check that each node has the same leaders and candidates
342 15.3 Find current leader and withdraw
343 15.4 Check that a new node was elected leader
344 15.5 Check that that new leader was the candidate of old leader
345 15.6 Run for election on old leader
346 15.7 Check that oldLeader is a candidate, and leader if only 1 node
347 15.8 Make sure that the old leader was added to the candidate list
348
349 old and new variable prefixes refer to data from before vs after
350 withdrawl and later before withdrawl vs after re-election
351 """
Devin Lim58046fa2017-07-05 16:55:00 -0700352 main.HA.isElectionFunctional( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700353
354 def CASE16( self, main ):
355 """
356 Install Distributed Primitives app
357 """
Devin Lim58046fa2017-07-05 16:55:00 -0700358 main.HA.installDistributedPrimitiveApp( main )
Jon Hall69b2b982016-05-11 12:04:59 -0700359
360 def CASE17( self, main ):
361 """
362 Check for basic functionality with distributed primitives
363 """
Devin Lim58046fa2017-07-05 16:55:00 -0700364 main.HA.checkDistPrimitivesFunc( main )