blob: be6833684d9e3715f38c2f723320bcb7330d56a2 [file] [log] [blame]
Jon Halla478b852017-12-04 15:00:15 -08001"""
2Copyright 2015 Open Networking Foundation (ONF)
3
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
11 (at your option) any later version.
12
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
21
22"""
23Description: This test is to determine if ONOS can handle
24 a minority of it's nodes restarting
25
26List of test cases:
27CASE1: Compile ONOS and push it to the test machines
28CASE2: Assign devices to controllers
29CASE21: Assign mastership to controllers
30CASE3: Assign intents
31CASE4: Ping across added host intents
32CASE5: Reading state of ONOS
33CASE61: The Failure inducing case.
34CASE62: The Failure recovery case.
35CASE7: Check state after control plane failure
36CASE8: Compare topo
37CASE9: Link s3-s28 down
38CASE10: Link s3-s28 up
39CASE11: Switch down
40CASE12: Switch up
41CASE13: Clean up
42CASE14: start election app on all onos nodes
43CASE15: Check that Leadership Election is still functional
44CASE16: Install Distributed Primitives app
45CASE17: Check for basic functionality with distributed primitives
46"""
47class HAupgradeRollback:
48
49 def __init__( self ):
50 self.default = ''
51
52 def CASE1( self, main ):
53 """
54 CASE1 is to compile ONOS and push it to the test machines
55
56 Startup sequence:
57 cell <name>
58 onos-verify-cell
59 NOTE: temporary - onos-remove-raft-logs
60 onos-uninstall
61 start mininet
62 git pull
63 mvn clean install
64 onos-package
65 onos-install -f
66 onos-wait-for-start
67 start cli sessions
68 start tcpdump
69 """
70 main.log.info( "ONOS HA test: Stop a minority of ONOS nodes - " +
71 "initialization" )
72 # These are for csv plotting in jenkins
73 main.HAlabels = []
74 main.HAdata = []
75 try:
76 from tests.dependencies.ONOSSetup import ONOSSetup
77 main.testSetUp = ONOSSetup()
78 except ImportError:
79 main.log.error( "ONOSSetup not found. exiting the test" )
80 main.cleanAndExit()
81 main.testSetUp.envSetupDescription()
82 try:
83 from tests.HA.dependencies.HA import HA
84 main.HA = HA()
85 cellName = main.params[ 'ENV' ][ 'cellName' ]
86 main.apps = main.params[ 'ENV' ][ 'appString' ]
87 stepResult = main.testSetUp.envSetup()
88 except Exception as e:
89 main.testSetUp.envSetupException( e )
90 main.testSetUp.evnSetupConclusion( stepResult )
91 main.HA.generateGraph( "HAupgrade" )
92
You Wanga0f6ff62018-01-11 15:46:30 -080093 main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
Jon Halla478b852017-12-04 15:00:15 -080094 extraApply=[ main.HA.startingMininet,
95 main.HA.copyBackupConfig ],
96 extraClean=main.HA.cleanUpGenPartition )
97
98 main.HA.initialSetUp( serviceClean=True )
99
100 def CASE2( self, main ):
101 """
102 Assign devices to controllers
103 """
104 main.HA.assignDevices( main )
105
106 def CASE21( self, main ):
107 """
108 Assign mastership to controllers
109 """
110 main.HA.assignMastership( main )
111
112 def CASE3( self, main ):
113 """
114 Assign intents
115 """
116 main.HA.assignIntents( main )
117
118 def CASE4( self, main ):
119 """
120 Ping across added host intents
121 """
122 main.HA.pingAcrossHostIntent( main )
123
124 def CASE5( self, main ):
125 """
126 Reading state of ONOS
127 """
128 main.HA.readingState( main )
129
130 def CASE60( self, main ):
131 """
132 Initialize the upgrade.
133 """
134 assert main, "main not defined"
135 assert utilities.assert_equals, "utilities.assert_equals not defined"
136 main.case( "Initialize upgrade" )
137 main.HA.upgradeInit( main )
138
139 def CASE61( self, main ):
140 """
141 Upgrade a minority of nodes PHASE 1
142 """
143 assert main, "main not defined"
144 assert utilities.assert_equals, "utilities.assert_equals not defined"
145 main.case( "Upgrade minority of ONOS nodes" )
146
147 main.step( "Checking ONOS Logs for errors" )
148 for ctrl in main.Cluster.active():
149 main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
150 main.log.warn( ctrl.checkLogs( ctrl.ipAddress ) )
151
152 main.kill = []
153 n = len( main.Cluster.runningNodes ) # Number of nodes
154 p = n / 2 # Number of nodes in the minority
155 for i in range( p ):
156 main.kill.append( main.Cluster.runningNodes[ i ] ) # ONOS node to kill, listed by index in main.nodes
157 main.HA.upgradeNodes( main )
158
159 main.step( "Checking ONOS nodes" )
160 nodeResults = utilities.retry( main.Cluster.nodesCheck,
161 False,
162 sleep=15,
163 attempts=5 )
164 utilities.assert_equals( expect=True, actual=nodeResults,
165 onpass="Nodes check successful",
166 onfail="Nodes check NOT successful" )
167
168 if not nodeResults:
169 for ctrl in main.Cluster.active():
170 main.log.debug( "{} components not ACTIVE: \n{}".format(
171 ctrl.name,
172 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
173 main.log.error( "Failed to start ONOS, stopping test" )
174 main.cleanAndExit()
175
176 def CASE62( self, main ):
177 """
178 Transfer to new version. PHASE 2
179 """
180 assert main, "main not defined"
181 assert utilities.assert_equals, "utilities.assert_equals not defined"
182 main.case( "Start the upgrade" )
183
184 main.step( "Send the command to switch to new version" )
185 ctrl = main.Cluster.next().CLI
186 upgraded = ctrl.issuUpgrade()
187 utilities.assert_equals( expect=main.TRUE, actual=upgraded,
188 onpass="Cluster has moved to the upgraded nodes",
189 onfail="Error transitioning to the upgraded nodes" )
190
191 main.step( "Check the status of the upgrade" )
192 ctrl = main.Cluster.next().CLI
193 status = ctrl.issu()
194 main.log.debug( status )
195 # TODO: check things here?
196
197 main.step( "Checking ONOS nodes" )
198 nodeResults = utilities.retry( main.Cluster.nodesCheck,
199 False,
200 sleep=15,
201 attempts=5 )
202 utilities.assert_equals( expect=True, actual=nodeResults,
203 onpass="Nodes check successful",
204 onfail="Nodes check NOT successful" )
205
206 def CASE63( self, main ):
207 """
208 Rollback the upgrade
209 """
210 main.case( "Rollback the upgrade" )
211 main.step( "Rollbak the upgrade" )
212 # send rollback command
213 ctrl = main.Cluster.next().CLI
214 rollback = ctrl.issuRollback()
215 utilities.assert_equals( expect=main.TRUE, actual=rollback,
216 onpass="Upgrade has been rolled back",
217 onfail="Error rolling back the upgrade" )
218
219 main.step( "Check the status of the upgrade" )
220 ctrl = main.Cluster.next().CLI
221 status = ctrl.issu()
222 main.log.debug( status )
223
224 # restart and reinstall old version on upgrade nodes
225 for ctrl in main.kill:
226 ctrl.onosStop( ctrl.ipAddress )
227 ctrl.onosUninstall( ctrl.ipAddress )
228 ctrl.onosInstall( options="-f", node=ctrl.ipAddress )
229 ctrl.onosSecureSSH( node=ctrl.ipAddress )
230 ctrl.startOnosCli( ctrl.ipAddress, waitForStart=True )
231 main.step( "Checking ONOS nodes" )
232 nodeResults = utilities.retry( main.Cluster.nodesCheck,
233 False,
234 sleep=15,
235 attempts=5 )
236 utilities.assert_equals( expect=True, actual=nodeResults,
237 onpass="Nodes check successful",
238 onfail="Nodes check NOT successful" )
239
240 if not nodeResults:
241 for ctrl in main.Cluster.active():
242 main.log.debug( "{} components not ACTIVE: \n{}".format(
243 ctrl.name,
244 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
245 main.log.error( "Failed to start ONOS, stopping test" )
246 main.cleanAndExit()
247
248 def CASE64( self, main ):
249 """
250 Reset the upgrade state.
251 """
252 assert main, "main not defined"
253 assert utilities.assert_equals, "utilities.assert_equals not defined"
254 main.case( "Reset the upgrade state" )
255
256 main.step( "Send the command to reset the upgrade" )
257 ctrl = main.Cluster.next().CLI
258 committed = ctrl.issuCommit()
259 utilities.assert_equals( expect=main.TRUE, actual=committed,
260 onpass="Upgrade has been committed",
261 onfail="Error committing the upgrade" )
262
263 main.step( "Check the status of the upgrade" )
264 ctrl = main.Cluster.next().CLI
265 status = ctrl.issu()
266 main.log.debug( status )
267 # TODO: check things here?
268
269 def CASE7( self, main ):
270 """
271 Check state after ONOS failure
272 """
273 try:
274 main.kill
275 except AttributeError:
276 main.kill = []
277
278 main.HA.checkStateAfterEvent( main, afterWhich=0 )
279 main.step( "Leadership Election is still functional" )
280 # Test of LeadershipElection
281 leaderList = []
282
283 restarted = []
284 for ctrl in main.kill:
285 restarted.append( ctrl.ipAddress )
286 leaderResult = main.TRUE
287
288 for ctrl in main.Cluster.active():
289 leaderN = ctrl.electionTestLeader()
290 leaderList.append( leaderN )
291 if leaderN == main.FALSE:
292 # error in response
293 main.log.error( "Something is wrong with " +
294 "electionTestLeader function, check the" +
295 " error logs" )
296 leaderResult = main.FALSE
297 elif leaderN is None:
298 main.log.error( ctrl.name +
299 " shows no leader for the election-app was" +
300 " elected after the old one died" )
301 leaderResult = main.FALSE
302 elif leaderN in restarted:
303 main.log.error( ctrl.name + " shows " + str( leaderN ) +
304 " as leader for the election-app, but it " +
305 "was restarted" )
306 leaderResult = main.FALSE
307 if len( set( leaderList ) ) != 1:
308 leaderResult = main.FALSE
309 main.log.error(
310 "Inconsistent view of leader for the election test app" )
311 # TODO: print the list
312 utilities.assert_equals(
313 expect=main.TRUE,
314 actual=leaderResult,
315 onpass="Leadership election passed",
316 onfail="Something went wrong with Leadership election" )
317
318 def CASE8( self, main ):
319 """
320 Compare topo
321 """
322 main.HA.compareTopo( main )
323
324 def CASE9( self, main ):
325 """
326 Link s3-s28 down
327 """
328 main.HA.linkDown( main )
329
330 def CASE10( self, main ):
331 """
332 Link s3-s28 up
333 """
334 main.HA.linkUp( main )
335
336 def CASE11( self, main ):
337 """
338 Switch Down
339 """
340 # NOTE: You should probably run a topology check after this
341 main.HA.switchDown( main )
342
343 def CASE12( self, main ):
344 """
345 Switch Up
346 """
347 # NOTE: You should probably run a topology check after this
348 main.HA.switchUp( main )
349
350 def CASE13( self, main ):
351 """
352 Clean up
353 """
354 main.HAlabels.append( "Restart" )
355 main.HAdata.append( str( main.restartTime ) )
356 main.HA.cleanUp( main )
357
358 def CASE14( self, main ):
359 """
360 start election app on all onos nodes
361 """
362 main.HA.startElectionApp( main )
363
364 def CASE15( self, main ):
365 """
366 Check that Leadership Election is still functional
367 15.1 Run election on each node
368 15.2 Check that each node has the same leaders and candidates
369 15.3 Find current leader and withdraw
370 15.4 Check that a new node was elected leader
371 15.5 Check that that new leader was the candidate of old leader
372 15.6 Run for election on old leader
373 15.7 Check that oldLeader is a candidate, and leader if only 1 node
374 15.8 Make sure that the old leader was added to the candidate list
375
376 old and new variable prefixes refer to data from before vs after
377 withdrawl and later before withdrawl vs after re-election
378 """
379 main.HA.isElectionFunctional( main )
380
381 def CASE16( self, main ):
382 """
383 Install Distributed Primitives app
384 """
385 main.HA.installDistributedPrimitiveApp( main )
386
387 def CASE17( self, main ):
388 """
389 Check for basic functionality with distributed primitives
390 """
391 main.HA.checkDistPrimitivesFunc( main )