blob: 93e65c9e19e390f3b3ff4ab64c77676b4b4a86dc [file] [log] [blame]
Jon Hall9ebd1bd2016-04-19 01:37:17 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2016 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070021"""
Jon Hall9ebd1bd2016-04-19 01:37:17 -070022Description: This test is to determine if ONOS can handle
23 dynamic scaling of the cluster size.
24
25List of test cases:
26CASE1: Compile ONOS and push it to the test machines
27CASE2: Assign devices to controllers
28CASE21: Assign mastership to controllers
29CASE3: Assign intents
30CASE4: Ping across added host intents
31CASE5: Reading state of ONOS
32CASE6: The scaling case.
33CASE7: Check state after control plane failure
34CASE8: Compare topo
35CASE9: Link s3-s28 down
36CASE10: Link s3-s28 up
37CASE11: Switch down
38CASE12: Switch up
39CASE13: Clean up
40CASE14: start election app on all onos nodes
41CASE15: Check that Leadership Election is still functional
42CASE16: Install Distributed Primitives app
43CASE17: Check for basic functionality with distributed primitives
44"""
Jon Hall9ebd1bd2016-04-19 01:37:17 -070045class HAscaling:
46
47 def __init__( self ):
48 self.default = ''
49
50 def CASE1( self, main ):
51 """
52 CASE1 is to compile ONOS and push it to the test machines
53
54 Startup sequence:
55 cell <name>
56 onos-verify-cell
57 NOTE: temporary - onos-remove-raft-logs
58 onos-uninstall
59 start mininet
60 git pull
61 mvn clean install
62 onos-package
63 onos-install -f
64 onos-wait-for-start
65 start cli sessions
66 start tcpdump
67 """
Jon Hall9ebd1bd2016-04-19 01:37:17 -070068 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
69 "initialization" )
Jon Hall9ebd1bd2016-04-19 01:37:17 -070070 # set global variables
71 # These are for csv plotting in jenkins
Devin Lim58046fa2017-07-05 16:55:00 -070072 main.HAlabels = []
73 main.HAdata = []
74 try:
75 from tests.dependencies.ONOSSetup import ONOSSetup
76 main.testSetUp = ONOSSetup()
77 except ImportError:
Jon Hall4f360bc2017-09-07 10:19:52 -070078 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070079 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070080 main.testSetUp.envSetupDescription()
Jon Halla478b852017-12-04 15:00:15 -080081 main.Cluster.numCtrls = 1
Jon Hall9ebd1bd2016-04-19 01:37:17 -070082 try:
83 from tests.HA.dependencies.HA import HA
84 main.HA = HA()
Devin Lim58046fa2017-07-05 16:55:00 -070085 from tests.HA.HAswapNodes.dependencies.Server import Server
Jon Hall9ebd1bd2016-04-19 01:37:17 -070086 main.Server = Server()
Devin Lim58046fa2017-07-05 16:55:00 -070087
88 # load some variables from the params file
89 cellName = main.params[ 'ENV' ][ 'cellName' ]
90 main.apps = main.params[ 'ENV' ][ 'appString' ]
Devin Lim142b5342017-07-20 15:22:39 -070091 stepResult = main.testSetUp.envSetup()
Jon Hall9ebd1bd2016-04-19 01:37:17 -070092 except Exception as e:
Devin Lim58046fa2017-07-05 16:55:00 -070093 main.testSetUp.envSetupException( e )
94 main.testSetUp.evnSetupConclusion( stepResult )
Jon Hall9ebd1bd2016-04-19 01:37:17 -070095
You Wanga0f6ff62018-01-11 15:46:30 -080096 main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
Jon Hall4f360bc2017-09-07 10:19:52 -070097 extraApply=[ main.HA.setServerForCluster,
98 main.HA.scalingMetadata,
99 main.HA.startingMininet,
100 main.HA.copyBackupConfig,
101 main.HA.setMetadataUrl ],
102 extraClean=main.HA.cleanUpOnosService,
103 installMax=True )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700104
Jon Hall4f360bc2017-09-07 10:19:52 -0700105 main.HA.initialSetUp( serviceClean=True )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700106
107 def CASE2( self, main ):
108 """
109 Assign devices to controllers
110 """
Devin Lim58046fa2017-07-05 16:55:00 -0700111 main.HA.assignDevices( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700112
113 def CASE21( self, main ):
114 """
115 Assign mastership to controllers
116 """
Devin Lim58046fa2017-07-05 16:55:00 -0700117 main.HA.assignMastership( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700118
119 def CASE3( self, main ):
120 """
121 Assign intents
122 """
Devin Lim58046fa2017-07-05 16:55:00 -0700123 main.HA.assignIntents( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700124
125 def CASE4( self, main ):
126 """
127 Ping across added host intents
128 """
Jon Hallca319892017-06-15 15:25:22 -0700129 main.HA.pingAcrossHostIntent( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700130
131 def CASE5( self, main ):
132 """
133 Reading state of ONOS
134 """
Devin Lim58046fa2017-07-05 16:55:00 -0700135 main.HA.readingState( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700136
137 def CASE6( self, main ):
138 """
139 The Scaling case.
140 """
141 import time
142 import re
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700143 assert main, "main not defined"
144 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700145 try:
Devin Lim58046fa2017-07-05 16:55:00 -0700146 main.HAlabels
147 except ( NameError, AttributeError ):
148 main.log.error( "main.HAlabels not defined, setting to []" )
149 main.HAlabels = []
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700150 try:
Devin Lim58046fa2017-07-05 16:55:00 -0700151 main.HAdata
152 except ( NameError, AttributeError ):
153 main.log.error( "main.HAdata not defined, setting to []" )
154 main.HAdata = []
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700155
Jon Hall69b2b982016-05-11 12:04:59 -0700156 main.case( "Scale the number of nodes in the ONOS cluster" )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700157
158 main.step( "Checking ONOS Logs for errors" )
Devin Lim142b5342017-07-20 15:22:39 -0700159 for ctrl in main.Cluster.active():
160 main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
161 main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700162
163 """
164 pop # of nodes from a list, might look like 1,3b,3,5b,5,7b,7,7b,5,5b,3...
165 modify cluster.json file appropriately
166 install/deactivate node as needed
167 """
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700168 try:
Devin Lim142b5342017-07-20 15:22:39 -0700169 prevNodes = main.Cluster.active()
Jon Hallf37d44d2017-05-24 10:37:30 -0700170 scale = main.scaling.pop( 0 )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700171 if "e" in scale:
172 equal = True
173 else:
174 equal = False
Devin Lim142b5342017-07-20 15:22:39 -0700175 main.Cluster.setRunningNode( int( re.search( "\d+", scale ).group( 0 ) ) )
Jon Hall2793f312018-02-02 11:16:46 -0800176 main.step( "Scaling to {} nodes".format( main.Cluster.numCtrls ) )
Devin Lim142b5342017-07-20 15:22:39 -0700177 genResult = main.Server.generateFile( main.Cluster.numCtrls, equal=equal )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700178 utilities.assert_equals( expect=main.TRUE, actual=genResult,
179 onpass="New cluster metadata file generated",
180 onfail="Failled to generate new metadata file" )
181 time.sleep( 5 ) # Give time for nodes to read new file
182 except IndexError:
Devin Lim44075962017-08-11 10:56:37 -0700183 main.cleanAndExit()
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700184
Devin Lim142b5342017-07-20 15:22:39 -0700185 activeNodes = [ i for i in range( 0, main.Cluster.numCtrls ) ]
186 newNodes = [ x for x in activeNodes if x not in prevNodes ]
187 main.Cluster.resetActive()
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700188 main.step( "Start new nodes" ) # OR stop old nodes?
189 started = main.TRUE
190 for i in newNodes:
Devin Lim142b5342017-07-20 15:22:39 -0700191 started = main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) and main.TRUE
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700192 utilities.assert_equals( expect=main.TRUE, actual=started,
193 onpass="ONOS started",
194 onfail="ONOS start NOT successful" )
195
Devin Lim142b5342017-07-20 15:22:39 -0700196 main.testSetUp.setupSsh( main.Cluster )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700197
Devin Lim142b5342017-07-20 15:22:39 -0700198 main.testSetUp.checkOnosService( main.Cluster )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700199
Devin Lim142b5342017-07-20 15:22:39 -0700200 main.Cluster.startCLIs()
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700201
202 main.step( "Checking ONOS nodes" )
Devin Lim3ebd5e72017-11-14 10:38:00 -0800203 nodeResults = utilities.retry( main.Cluster.nodesCheck,
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700204 False,
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700205 attempts=5 )
206 utilities.assert_equals( expect=True, actual=nodeResults,
207 onpass="Nodes check successful",
208 onfail="Nodes check NOT successful" )
209
210 for i in range( 10 ):
211 ready = True
Devin Lim142b5342017-07-20 15:22:39 -0700212 for ctrl in main.Cluster.active():
213 output = ctrl.CLI.summary()
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700214 if not output:
215 ready = False
216 if ready:
217 break
218 time.sleep( 30 )
219 utilities.assert_equals( expect=True, actual=ready,
220 onpass="ONOS summary command succeded",
221 onfail="ONOS summary command failed" )
222 if not ready:
Devin Lim44075962017-08-11 10:56:37 -0700223 main.cleanAndExit()
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700224
225 # Rerun for election on new nodes
226 runResults = main.TRUE
Devin Lim142b5342017-07-20 15:22:39 -0700227 for ctrl in main.Cluster.active():
228 run = ctrl.CLI.electionTestRun()
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700229 if run != main.TRUE:
Devin Lim142b5342017-07-20 15:22:39 -0700230 main.log.error( "Error running for election on " + ctrl.name )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700231 runResults = runResults and run
232 utilities.assert_equals( expect=main.TRUE, actual=runResults,
233 onpass="Reran for election",
234 onfail="Failed to rerun for election" )
235
236 # TODO: Make this configurable
237 time.sleep( 60 )
Devin Lim142b5342017-07-20 15:22:39 -0700238 main.HA.commonChecks()
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700239
240 def CASE7( self, main ):
241 """
242 Check state after ONOS scaling
243 """
Devin Lim142b5342017-07-20 15:22:39 -0700244 main.HA.checkStateAfterEvent( main, afterWhich=1 )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700245
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700246 main.step( "Leadership Election is still functional" )
247 # Test of LeadershipElection
248 leaderList = []
249 leaderResult = main.TRUE
250
Devin Lim142b5342017-07-20 15:22:39 -0700251 for ctrl in main.Cluster.active():
252 leaderN = ctrl.CLI.electionTestLeader()
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700253 leaderList.append( leaderN )
254 if leaderN == main.FALSE:
255 # error in response
256 main.log.error( "Something is wrong with " +
257 "electionTestLeader function, check the" +
258 " error logs" )
259 leaderResult = main.FALSE
260 elif leaderN is None:
Jon Halla478b852017-12-04 15:00:15 -0800261 main.log.error( ctrl.name +
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700262 " shows no leader for the election-app." )
263 leaderResult = main.FALSE
264 if len( set( leaderList ) ) != 1:
265 leaderResult = main.FALSE
266 main.log.error(
267 "Inconsistent view of leader for the election test app" )
268 # TODO: print the list
269 utilities.assert_equals(
270 expect=main.TRUE,
271 actual=leaderResult,
272 onpass="Leadership election passed",
273 onfail="Something went wrong with Leadership election" )
274
275 def CASE8( self, main ):
276 """
277 Compare topo
278 """
Devin Lim58046fa2017-07-05 16:55:00 -0700279 main.HA.compareTopo( main )
Jon Halld2871c22016-07-26 11:01:14 -0700280
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700281 def CASE9( self, main ):
282 """
283 Link s3-s28 down
284 """
Devin Lim58046fa2017-07-05 16:55:00 -0700285 main.HA.linkDown( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700286
287 def CASE10( self, main ):
288 """
289 Link s3-s28 up
290 """
Devin Lim58046fa2017-07-05 16:55:00 -0700291 main.HA.linkUp( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700292
293 def CASE11( self, main ):
294 """
295 Switch Down
296 """
297 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700298 main.HA.switchDown( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700299
300 def CASE12( self, main ):
301 """
302 Switch Up
303 """
304 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700305 main.HA.switchUp( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700306
307 def CASE13( self, main ):
308 """
309 Clean up
310 """
Devin Lim58046fa2017-07-05 16:55:00 -0700311 main.HA.cleanUp( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700312
313 main.step( "Stopping webserver" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700314 status = main.Server.stop()
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700315 utilities.assert_equals( expect=main.TRUE, actual=status,
316 onpass="Stop Server",
317 onfail="Failled to stop SimpleHTTPServer" )
318 del main.Server
319
320 def CASE14( self, main ):
321 """
322 start election app on all onos nodes
323 """
Devin Lim58046fa2017-07-05 16:55:00 -0700324 main.HA.startElectionApp( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700325
326 def CASE15( self, main ):
327 """
328 Check that Leadership Election is still functional
329 15.1 Run election on each node
330 15.2 Check that each node has the same leaders and candidates
331 15.3 Find current leader and withdraw
332 15.4 Check that a new node was elected leader
333 15.5 Check that that new leader was the candidate of old leader
334 15.6 Run for election on old leader
335 15.7 Check that oldLeader is a candidate, and leader if only 1 node
336 15.8 Make sure that the old leader was added to the candidate list
337
338 old and new variable prefixes refer to data from before vs after
339 withdrawl and later before withdrawl vs after re-election
340 """
Devin Lim58046fa2017-07-05 16:55:00 -0700341 main.HA.isElectionFunctional( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700342
343 def CASE16( self, main ):
344 """
345 Install Distributed Primitives app
346 """
Devin Lim58046fa2017-07-05 16:55:00 -0700347 main.HA.installDistributedPrimitiveApp( main )
Jon Hall9ebd1bd2016-04-19 01:37:17 -0700348
349 def CASE17( self, main ):
350 """
351 Check for basic functionality with distributed primitives
352 """
Devin Lim58046fa2017-07-05 16:55:00 -0700353 main.HA.checkDistPrimitivesFunc( main )