blob: 2460c94264a9c462efbf5380866875ecbecfe546 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2015 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070021"""
Jon Hall5cf14d52015-07-16 12:15:19 -070022Description: This test is to determine if ONOS can handle
23 all of it's nodes restarting
24
25List of test cases:
26CASE1: Compile ONOS and push it to the test machines
27CASE2: Assign devices to controllers
28CASE21: Assign mastership to controllers
29CASE3: Assign intents
30CASE4: Ping across added host intents
31CASE5: Reading state of ONOS
32CASE6: The Failure case.
33CASE7: Check state after control plane failure
34CASE8: Compare topo
35CASE9: Link s3-s28 down
36CASE10: Link s3-s28 up
37CASE11: Switch down
38CASE12: Switch up
39CASE13: Clean up
40CASE14: start election app on all onos nodes
41CASE15: Check that Leadership Election is still functional
42CASE16: Install Distributed Primitives app
43CASE17: Check for basic functionality with distributed primitives
44"""
Jon Hall5cf14d52015-07-16 12:15:19 -070045class HAclusterRestart:
46
47 def __init__( self ):
48 self.default = ''
49
50 def CASE1( self, main ):
51 """
52 CASE1 is to compile ONOS and push it to the test machines
53
54 Startup sequence:
55 cell <name>
56 onos-verify-cell
57 NOTE: temporary - onos-remove-raft-logs
58 onos-uninstall
59 start mininet
60 git pull
61 mvn clean install
62 onos-package
63 onos-install -f
64 onos-wait-for-start
65 start cli sessions
66 start tcpdump
67 """
68 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
69 "initialization" )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 # These are for csv plotting in jenkins
Devin Lim58046fa2017-07-05 16:55:00 -070071 main.HAlabels = []
72 main.HAdata = []
73 try:
74 from tests.dependencies.ONOSSetup import ONOSSetup
75 main.testSetUp = ONOSSetup()
76 except ImportError:
77 main.log.error( "ONOSSetup not found exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070078 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070079 main.testSetUp.envSetupDescription()
Jon Halle1a3b752015-07-22 13:02:46 -070080 try:
Jon Hall53c5e662016-04-13 16:06:56 -070081 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070082 main.HA = HA()
Devin Lim58046fa2017-07-05 16:55:00 -070083 # load some variables from the params file
84 cellName = main.params[ 'ENV' ][ 'cellName' ]
85 main.apps = main.params[ 'ENV' ][ 'appString' ]
Devin Lim142b5342017-07-20 15:22:39 -070086 stepResult = main.testSetUp.envSetup()
Jon Halle1a3b752015-07-22 13:02:46 -070087 except Exception as e:
Devin Lim58046fa2017-07-05 16:55:00 -070088 main.testSetUp.envSetupException( e )
89 main.testSetUp.evnSetupConclusion( stepResult )
Jon Halle1a3b752015-07-22 13:02:46 -070090
Devin Lim142b5342017-07-20 15:22:39 -070091 main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName=cellName, removeLog=True,
Devin Lim58046fa2017-07-05 16:55:00 -070092 extraApply=main.HA.startingMininet )
Jon Hall5cf14d52015-07-16 12:15:19 -070093
Devin Lim58046fa2017-07-05 16:55:00 -070094 main.HA.initialSetUp()
Jon Hall9d2dcad2016-04-08 10:15:20 -070095
Jon Hall5cf14d52015-07-16 12:15:19 -070096 def CASE2( self, main ):
97 """
98 Assign devices to controllers
99 """
Devin Lim58046fa2017-07-05 16:55:00 -0700100 main.HA.assignDevices( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700101
102 def CASE21( self, main ):
103 """
104 Assign mastership to controllers
105 """
Devin Lim58046fa2017-07-05 16:55:00 -0700106 main.HA.assignMastership( main )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700107
Jon Hall5cf14d52015-07-16 12:15:19 -0700108 def CASE3( self, main ):
109 """
110 Assign intents
111 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700112 main.HA.assignIntents( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700113
114 def CASE4( self, main ):
115 """
116 Ping across added host intents
117 """
Jon Hallca319892017-06-15 15:25:22 -0700118 main.HA.pingAcrossHostIntent( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700119
120 def CASE5( self, main ):
121 """
122 Reading state of ONOS
123 """
Devin Lim58046fa2017-07-05 16:55:00 -0700124 main.HA.readingState( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700125
126 def CASE6( self, main ):
127 """
128 The Failure case.
129 """
130 import time
Jon Hall5cf14d52015-07-16 12:15:19 -0700131 assert main, "main not defined"
132 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700133 try:
Devin Lim58046fa2017-07-05 16:55:00 -0700134 main.HAlabels
135 except ( NameError, AttributeError ):
136 main.log.error( "main.HAlabels not defined, setting to []" )
137 main.HAlabels = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700138 try:
Devin Lim58046fa2017-07-05 16:55:00 -0700139 main.HAdata
140 except ( NameError, AttributeError ):
141 main.log.error( "main.HAdata not defined, setting to []" )
142 main.HAdata = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700143
144 main.case( "Restart entire ONOS cluster" )
145
Jon Hall5ec6b1b2015-09-17 18:20:14 -0700146 main.step( "Checking ONOS Logs for errors" )
Devin Lim142b5342017-07-20 15:22:39 -0700147 for ctrl in main.Cluster.active():
148 main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
149 main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
Devin Lim6301b742017-08-07 11:00:21 -0700150 killTime = time.time()
Devin Lim142b5342017-07-20 15:22:39 -0700151 main.testSetUp.killingAllOnos( main.Cluster, True, False )
Jon Hall5cf14d52015-07-16 12:15:19 -0700152
Devin Lim142b5342017-07-20 15:22:39 -0700153 main.testSetUp.checkOnosService( main.Cluster )
Jon Hall5cf14d52015-07-16 12:15:19 -0700154
Devin Lim142b5342017-07-20 15:22:39 -0700155 main.testSetUp.startOnosClis( main.Cluster )
Jon Hall5cf14d52015-07-16 12:15:19 -0700156
Devin Lim142b5342017-07-20 15:22:39 -0700157 ready = utilities.retry( main.Cluster.command,
158 False,
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700159 kwargs={ "function": "summary", "contentCheck": True },
Devin Lim142b5342017-07-20 15:22:39 -0700160 sleep=30,
161 attempts=10 )
Jon Hall6e709752016-02-01 13:38:46 -0800162 utilities.assert_equals( expect=True, actual=ready,
163 onpass="ONOS summary command succeded",
164 onfail="ONOS summary command failed" )
165 if not ready:
Devin Lim44075962017-08-11 10:56:37 -0700166 main.cleanAndExit()
Jon Hall6e709752016-02-01 13:38:46 -0800167
Jon Hall5cf14d52015-07-16 12:15:19 -0700168 # Grab the time of restart so we chan check how long the gossip
169 # protocol has had time to work
170 main.restartTime = time.time() - killTime
171 main.log.debug( "Restart time: " + str( main.restartTime ) )
Devin Lim58046fa2017-07-05 16:55:00 -0700172 main.HAlabels.append( "Restart" )
173 main.HAdata.append( str( main.restartTime ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700174
Jon Hall5cf14d52015-07-16 12:15:19 -0700175 # Rerun for election on restarted nodes
Devin Lim142b5342017-07-20 15:22:39 -0700176 runResults = main.Cluster.command( "electionTestRun", returnBool=True )
177 utilities.assert_equals( expect=True, actual=runResults,
Jon Hall5cf14d52015-07-16 12:15:19 -0700178 onpass="Reran for election",
179 onfail="Failed to rerun for election" )
180
181 # TODO: Make this configurable
182 time.sleep( 60 )
Devin Lim142b5342017-07-20 15:22:39 -0700183
184 main.HA.commonChecks()
Jon Hall5cf14d52015-07-16 12:15:19 -0700185
186 def CASE7( self, main ):
187 """
188 Check state after ONOS failure
189 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700190 # NOTE: Store has no durability, so intents are lost across system
191 # restarts
Devin Lim142b5342017-07-20 15:22:39 -0700192 main.HA.checkStateAfterEvent( main, afterWhich=0, isRestart=True )
Jon Hall5cf14d52015-07-16 12:15:19 -0700193
Jon Hall5cf14d52015-07-16 12:15:19 -0700194 main.step( "Leadership Election is still functional" )
195 # Test of LeadershipElection
196 leaderList = []
197 leaderResult = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700198
Devin Lim142b5342017-07-20 15:22:39 -0700199 for ctrl in main.Cluster.active():
200 ctrl.CLI.electionTestLeader()
201 leaderN = ctrl.CLI.electionTestLeader()
Jon Hall5cf14d52015-07-16 12:15:19 -0700202 leaderList.append( leaderN )
203 if leaderN == main.FALSE:
204 # error in response
205 main.log.error( "Something is wrong with " +
206 "electionTestLeader function, check the" +
207 " error logs" )
208 leaderResult = main.FALSE
209 elif leaderN is None:
Jon Halla478b852017-12-04 15:00:15 -0800210 main.log.error( ctrl.name +
Jon Hall5cf14d52015-07-16 12:15:19 -0700211 " shows no leader for the election-app." )
212 leaderResult = main.FALSE
213 if len( set( leaderList ) ) != 1:
214 leaderResult = main.FALSE
215 main.log.error(
216 "Inconsistent view of leader for the election test app" )
217 # TODO: print the list
218 utilities.assert_equals(
219 expect=main.TRUE,
220 actual=leaderResult,
221 onpass="Leadership election passed",
222 onfail="Something went wrong with Leadership election" )
223
224 def CASE8( self, main ):
225 """
226 Compare topo
227 """
Devin Lim58046fa2017-07-05 16:55:00 -0700228 main.HA.compareTopo( main )
Jon Halld2871c22016-07-26 11:01:14 -0700229
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 def CASE9( self, main ):
231 """
232 Link s3-s28 down
233 """
Devin Lim58046fa2017-07-05 16:55:00 -0700234 main.HA.linkDown( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700235
236 def CASE10( self, main ):
237 """
238 Link s3-s28 up
239 """
Devin Lim58046fa2017-07-05 16:55:00 -0700240 main.HA.linkUp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700241
242 def CASE11( self, main ):
243 """
244 Switch Down
245 """
246 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700247 main.HA.switchDown( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700248
249 def CASE12( self, main ):
250 """
251 Switch Up
252 """
253 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700254 main.HA.switchUp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700255
256 def CASE13( self, main ):
257 """
258 Clean up
259 """
Devin Lim58046fa2017-07-05 16:55:00 -0700260 main.HA.cleanUp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700261
262 def CASE14( self, main ):
263 """
264 start election app on all onos nodes
265 """
Devin Lim58046fa2017-07-05 16:55:00 -0700266 main.HA.startElectionApp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700267
268 def CASE15( self, main ):
269 """
270 Check that Leadership Election is still functional
acsmars9475b1c2015-08-28 18:02:08 -0700271 15.1 Run election on each node
272 15.2 Check that each node has the same leaders and candidates
273 15.3 Find current leader and withdraw
274 15.4 Check that a new node was elected leader
275 15.5 Check that that new leader was the candidate of old leader
276 15.6 Run for election on old leader
277 15.7 Check that oldLeader is a candidate, and leader if only 1 node
278 15.8 Make sure that the old leader was added to the candidate list
279
280 old and new variable prefixes refer to data from before vs after
281 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -0700282 """
Devin Lim58046fa2017-07-05 16:55:00 -0700283 main.HA.isElectionFunctional( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700284
285 def CASE16( self, main ):
286 """
287 Install Distributed Primitives app
288 """
Devin Lim58046fa2017-07-05 16:55:00 -0700289 main.HA.installDistributedPrimitiveApp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700290
291 def CASE17( self, main ):
292 """
293 Check for basic functionality with distributed primitives
294 """
Devin Lim58046fa2017-07-05 16:55:00 -0700295 main.HA.checkDistPrimitivesFunc( main )