blob: 888393b13e9d4032cca9a88965da1749188418e5 [file] [log] [blame]
Jon Hall7ce46ea2018-02-05 12:20:59 -08001"""
2Copyright 2018 Open Networking Foundation ( ONF )
3
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
11 ( at your option ) any later version.
12
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
21"""
22Description: This test is to determine if ONOS can handle
23 all of it's nodes restarting
24
25List of test cases:
26CASE1: Compile ONOS and push it to the test machines
27CASE2: Assign devices to controllers
28CASE21: Assign mastership to controllers
29CASE3: Assign intents
30CASE4: Ping across added host intents
31CASE5: Reading state of ONOS
32CASE6: The Failure case.
33CASE7: Check state after control plane failure
34CASE8: Compare topo
35CASE9: Link s3-s28 down
36CASE10: Link s3-s28 up
37CASE11: Switch down
38CASE12: Switch up
39CASE13: Clean up
40CASE14: start election app on all onos nodes
41CASE15: Check that Leadership Election is still functional
42CASE16: Install Distributed Primitives app
43CASE17: Check for basic functionality with distributed primitives
44"""
45class HAbackupRecover:
46
47 def __init__( self ):
48 self.default = ''
49
50 def CASE1( self, main ):
51 """
52 CASE1 is to compile ONOS and push it to the test machines
53
54 Startup sequence:
55 cell <name>
56 onos-verify-cell
57 NOTE: temporary - onos-remove-raft-logs
58 onos-uninstall
59 start mininet
60 git pull
61 mvn clean install
62 onos-package
63 onos-install -f
64 onos-wait-for-start
65 start cli sessions
66 start tcpdump
67 """
68 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
69 "initialization" )
70 # These are for csv plotting in jenkins
71 main.HAlabels = []
72 main.HAdata = []
73 try:
74 from tests.dependencies.ONOSSetup import ONOSSetup
75 main.testSetUp = ONOSSetup()
76 except ImportError:
Jon Hallab611372018-02-21 15:26:05 -080077 main.log.error( "ONOSSetup not found. exiting the test" )
Jon Hall7ce46ea2018-02-05 12:20:59 -080078 main.cleanAndExit()
79 main.testSetUp.envSetupDescription()
80 try:
81 from tests.HA.dependencies.HA import HA
82 main.HA = HA()
83 # load some variables from the params file
84 cellName = main.params[ 'ENV' ][ 'cellName' ]
85 main.apps = main.params[ 'ENV' ][ 'appString' ]
Jon Hallab611372018-02-21 15:26:05 -080086 stepResult = main.testSetUp.envSetup( includeCaseDesc=False )
Jon Hall7ce46ea2018-02-05 12:20:59 -080087 except Exception as e:
88 main.testSetUp.envSetupException( e )
Jon Hallaa1d9b82020-07-30 13:49:42 -070089 main.testSetUp.envSetupConclusion( stepResult )
Jon Hall7ce46ea2018-02-05 12:20:59 -080090
Jon Hall5a5c8432018-11-28 11:39:57 -080091 applyFuncs = [ main.HA.removeKarafConsoleLogging ]
92
Jon Hallab611372018-02-21 15:26:05 -080093 try:
94 if main.params[ 'topology' ][ 'topoFile' ]:
95 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
Jon Hallab611372018-02-21 15:26:05 -080096 else:
Jon Hall5a5c8432018-11-28 11:39:57 -080097 applyFuncs.append( main.HA.startingMininet )
Jon Hallab611372018-02-21 15:26:05 -080098 except (KeyError, IndexError):
Jon Hall5a5c8432018-11-28 11:39:57 -080099 applyFuncs.append( main.HA.startingMininet )
Jon Hallab611372018-02-21 15:26:05 -0800100
Jon Hall3e6edb32018-08-21 16:20:30 -0700101 main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, extraApply=applyFuncs )
Jon Hall7ce46ea2018-02-05 12:20:59 -0800102
103 main.HA.initialSetUp()
104
Jon Hallab611372018-02-21 15:26:05 -0800105 main.step( 'Set logging levels' )
106 logging = True
107 try:
108 logs = main.params.get( 'ONOS_Logging', False )
109 if logs:
110 for namespace, level in logs.items():
111 for ctrl in main.Cluster.active():
112 ctrl.CLI.logSet( level, namespace )
113 except AttributeError:
114 logging = False
115 utilities.assert_equals( expect=True, actual=logging,
116 onpass="Set log levels",
117 onfail="Failed to set log levels" )
118
Jon Hall7ce46ea2018-02-05 12:20:59 -0800119 def CASE2( self, main ):
120 """
121 Assign devices to controllers
122 """
123 main.HA.assignDevices( main )
124
Jon Hallab611372018-02-21 15:26:05 -0800125 def CASE102( self, main ):
126 """
127 Set up Spine-Leaf fabric topology in Mininet
128 """
129 main.HA.startTopology( main )
130
Jon Hall7ce46ea2018-02-05 12:20:59 -0800131 def CASE21( self, main ):
132 """
133 Assign mastership to controllers
134 """
135 main.HA.assignMastership( main )
136
137 def CASE3( self, main ):
138 """
139 Assign intents
140 """
141 main.HA.assignIntents( main )
142
143 def CASE4( self, main ):
144 """
145 Ping across added host intents
146 """
147 main.HA.pingAcrossHostIntent( main )
148
Jon Hallab611372018-02-21 15:26:05 -0800149 def CASE104( self, main ):
150 """
151 Ping Hosts
152 """
153 main.case( "Check connectivity" )
154 main.step( "Ping between all hosts" )
155 pingResult = main.Mininet1.pingall()
156 utilities.assert_equals( expect=main.TRUE, actual=pingResult,
157 onpass="All Pings Passed",
158 onfail="Failed to ping between all hosts" )
159
Jon Hall7ce46ea2018-02-05 12:20:59 -0800160 def CASE5( self, main ):
161 """
162 Reading state of ONOS
163 """
164 main.HA.readingState( main )
165
166 def CASE6( self, main ):
167 """
168 The Failure case.
169 """
170 import time
171 assert main, "main not defined"
172 assert utilities.assert_equals, "utilities.assert_equals not defined"
173 try:
174 main.HAlabels
175 except ( NameError, AttributeError ):
176 main.log.error( "main.HAlabels not defined, setting to []" )
177 main.HAlabels = []
178 try:
179 main.HAdata
180 except ( NameError, AttributeError ):
181 main.log.error( "main.HAdata not defined, setting to []" )
182 main.HAdata = []
183
184 main.case( "Restart entire ONOS cluster with backed up state" )
185
186 main.step( "Backup ONOS data" )
187 location = "/tmp/" + main.TEST + ".tar.gz"
188 backupResult = main.HA.backupData( main, location )
189 utilities.assert_equals( expect=True, actual=backupResult,
190 onpass="ONOS backup succeded",
191 onfail="ONOS backup failed" )
192
193 main.step( "Checking ONOS Logs for errors" )
194 for ctrl in main.Cluster.active():
195 main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
196 main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
197
198 killTime = time.time()
199 main.testSetUp.uninstallOnos( main.Cluster, uninstallMax=True )
200
201 clusterSize = len( main.Cluster.active() )
202 main.Cluster.setRunningNode( 0 ) # So we can install without starting ONOS
203 main.testSetUp.installOnos( main.Cluster, installMax=True )
204 main.Cluster.setRunningNode( clusterSize )
205
206 main.step( "Restore ONOS data" )
207 restoreResult = main.HA.restoreData( main, location )
208 utilities.assert_equals( expect=True, actual=restoreResult,
209 onpass="ONOS restore succeded",
210 onfail="ONOS restore failed" )
211
212 main.step( "Restart ONOS nodes" )
213 started = main.Cluster.command( "onosStart",
214 args=[ "ipAddress" ],
215 getFrom=0,
216 funcFromCtrl=True )
217 for ctrl in main.Cluster.controllers:
218 ctrl.active = True
219 main.log.debug( repr( ctrl ) )
220
221 main.testSetUp.setupSsh( main.Cluster )
222 main.testSetUp.checkOnosService( main.Cluster )
223 main.testSetUp.startOnosClis( main.Cluster )
224
225 ready = utilities.retry( main.Cluster.command,
226 False,
227 kwargs={ "function": "summary", "contentCheck": True },
228 sleep=30,
229 attempts=10 )
230 utilities.assert_equals( expect=True, actual=ready,
231 onpass="ONOS summary command succeded",
232 onfail="ONOS summary command failed" )
233 if not ready:
234 main.cleanAndExit()
235
236 # Grab the time of restart so we chan check how long the gossip
237 # protocol has had time to work
238 main.restartTime = time.time() - killTime
239 main.log.debug( "Restart time: " + str( main.restartTime ) )
240 main.HAlabels.append( "Restart" )
241 main.HAdata.append( str( main.restartTime ) )
242
243 # Rerun for election on restarted nodes
244 runResults = main.Cluster.command( "electionTestRun", returnBool=True )
245 utilities.assert_equals( expect=True, actual=runResults,
246 onpass="Reran for election",
247 onfail="Failed to rerun for election" )
248
249 main.HA.commonChecks()
Jon Hall9677ed32018-04-24 11:16:23 -0700250 time.sleep(60)
Jon Hall7ce46ea2018-02-05 12:20:59 -0800251
252 def CASE7( self, main ):
253 """
254 Check state after ONOS failure
255 """
256 # NOTE: Store has no durability, so intents are lost across system
257 # restarts
258 main.HA.checkStateAfterEvent( main, afterWhich=0, isRestart=True )
259
260 main.step( "Leadership Election is still functional" )
261 # Test of LeadershipElection
262 leaderList = []
263 leaderResult = main.TRUE
264
265 for ctrl in main.Cluster.active():
Jon Hall7ce46ea2018-02-05 12:20:59 -0800266 leaderN = ctrl.CLI.electionTestLeader()
267 leaderList.append( leaderN )
268 if leaderN == main.FALSE:
269 # error in response
270 main.log.error( "Something is wrong with " +
271 "electionTestLeader function, check the" +
272 " error logs" )
273 leaderResult = main.FALSE
274 elif leaderN is None:
275 main.log.error( ctrl.name +
276 " shows no leader for the election-app." )
277 leaderResult = main.FALSE
278 if len( set( leaderList ) ) != 1:
279 leaderResult = main.FALSE
280 main.log.error(
281 "Inconsistent view of leader for the election test app" )
Jon Hallab611372018-02-21 15:26:05 -0800282 main.log.debug( leaderList )
Jon Hall7ce46ea2018-02-05 12:20:59 -0800283 utilities.assert_equals(
284 expect=main.TRUE,
285 actual=leaderResult,
286 onpass="Leadership election passed",
287 onfail="Something went wrong with Leadership election" )
288
289 def CASE8( self, main ):
290 """
291 Compare topo
292 """
293 main.HA.compareTopo( main )
294
295 def CASE9( self, main ):
296 """
Jon Hallab611372018-02-21 15:26:05 -0800297 Link down
Jon Hall7ce46ea2018-02-05 12:20:59 -0800298 """
Jon Hallab611372018-02-21 15:26:05 -0800299 src = main.params['kill']['linkSrc']
300 dst = main.params['kill']['linkDst']
301 main.HA.linkDown( main, src, dst )
Jon Hall7ce46ea2018-02-05 12:20:59 -0800302
303 def CASE10( self, main ):
304 """
Jon Hallab611372018-02-21 15:26:05 -0800305 Link up
Jon Hall7ce46ea2018-02-05 12:20:59 -0800306 """
Jon Hallab611372018-02-21 15:26:05 -0800307 src = main.params['kill']['linkSrc']
308 dst = main.params['kill']['linkDst']
309 main.HA.linkUp( main, src, dst )
Jon Hall7ce46ea2018-02-05 12:20:59 -0800310
311 def CASE11( self, main ):
312 """
313 Switch Down
314 """
315 # NOTE: You should probably run a topology check after this
316 main.HA.switchDown( main )
317
318 def CASE12( self, main ):
319 """
320 Switch Up
321 """
322 # NOTE: You should probably run a topology check after this
323 main.HA.switchUp( main )
324
325 def CASE13( self, main ):
326 """
327 Clean up
328 """
329 main.HA.cleanUp( main )
330
331 def CASE14( self, main ):
332 """
Jon Hallab611372018-02-21 15:26:05 -0800333 Start election app on all onos nodes
Jon Hall7ce46ea2018-02-05 12:20:59 -0800334 """
335 try:
336 main.HA.startElectionApp( main )
337 except Exception as e:
338 main.log.error( e )
339
340 def CASE15( self, main ):
341 """
342 Check that Leadership Election is still functional
343 15.1 Run election on each node
344 15.2 Check that each node has the same leaders and candidates
345 15.3 Find current leader and withdraw
346 15.4 Check that a new node was elected leader
347 15.5 Check that that new leader was the candidate of old leader
348 15.6 Run for election on old leader
349 15.7 Check that oldLeader is a candidate, and leader if only 1 node
350 15.8 Make sure that the old leader was added to the candidate list
351
352 old and new variable prefixes refer to data from before vs after
353 withdrawl and later before withdrawl vs after re-election
354 """
355 main.HA.isElectionFunctional( main )
356
357 def CASE16( self, main ):
358 """
359 Install Distributed Primitives app
360 """
361 main.HA.installDistributedPrimitiveApp( main )
362
363 def CASE17( self, main ):
364 """
365 Check for basic functionality with distributed primitives
366 """
367 main.HA.checkDistPrimitivesFunc( main )