blob: cd47131a42b13f4ffe70b253a8f25da879868f92 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2015 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070021"""
Jon Hall5cf14d52015-07-16 12:15:19 -070022Description: This test is to determine if ONOS can handle
23 a minority of it's nodes restarting
24
25List of test cases:
26CASE1: Compile ONOS and push it to the test machines
27CASE2: Assign devices to controllers
28CASE21: Assign mastership to controllers
29CASE3: Assign intents
30CASE4: Ping across added host intents
31CASE5: Reading state of ONOS
Jon Hallb3ed8ed2015-10-28 16:43:55 -070032CASE61: The Failure inducing case.
33CASE62: The Failure recovery case.
Jon Hall5cf14d52015-07-16 12:15:19 -070034CASE7: Check state after control plane failure
35CASE8: Compare topo
36CASE9: Link s3-s28 down
37CASE10: Link s3-s28 up
38CASE11: Switch down
39CASE12: Switch up
40CASE13: Clean up
41CASE14: start election app on all onos nodes
42CASE15: Check that Leadership Election is still functional
43CASE16: Install Distributed Primitives app
44CASE17: Check for basic functionality with distributed primitives
45"""
Jon Hallb3ed8ed2015-10-28 16:43:55 -070046class HAkillNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -070047
48 def __init__( self ):
49 self.default = ''
50
51 def CASE1( self, main ):
52 """
53 CASE1 is to compile ONOS and push it to the test machines
54
55 Startup sequence:
56 cell <name>
57 onos-verify-cell
58 NOTE: temporary - onos-remove-raft-logs
59 onos-uninstall
60 start mininet
61 git pull
62 mvn clean install
63 onos-package
64 onos-install -f
65 onos-wait-for-start
66 start cli sessions
67 start tcpdump
68 """
Jon Hall6e709752016-02-01 13:38:46 -080069 main.log.info( "ONOS HA test: Restart a minority of ONOS nodes - " +
Jon Hall5cf14d52015-07-16 12:15:19 -070070 "initialization" )
Jon Halla440e872016-03-31 15:15:50 -070071 # These are for csv plotting in jenkins
Devin Lim58046fa2017-07-05 16:55:00 -070072 main.HAlabels = []
73 main.HAdata = []
74 try:
75 from tests.dependencies.ONOSSetup import ONOSSetup
76 main.testSetUp = ONOSSetup()
77 except ImportError:
78 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070079 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070080 main.testSetUp.envSetupDescription()
Jon Halle1a3b752015-07-22 13:02:46 -070081 try:
Jon Hall53c5e662016-04-13 16:06:56 -070082 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070083 main.HA = HA()
Devin Lim58046fa2017-07-05 16:55:00 -070084 cellName = main.params[ 'ENV' ][ 'cellName' ]
85 main.apps = main.params[ 'ENV' ][ 'appString' ]
Devin Lim142b5342017-07-20 15:22:39 -070086 stepResult = main.testSetUp.envSetup()
Jon Halle1a3b752015-07-22 13:02:46 -070087 except Exception as e:
Devin Lim58046fa2017-07-05 16:55:00 -070088 main.testSetUp.envSetupException( e )
89 main.testSetUp.evnSetupConclusion( stepResult )
Jon Hall5cf14d52015-07-16 12:15:19 -070090
Jon Hallca319892017-06-15 15:25:22 -070091 main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName=cellName, removeLog=True,
Jon Hall4f360bc2017-09-07 10:19:52 -070092 extraApply=[ main.HA.startingMininet,
93 main.HA.customizeOnosGenPartitions,
94 main.HA.copyBackupConfig,
95 main.ONOSbench.preventAutoRespawn ],
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070096 extraClean=main.HA.cleanUpGenPartition )
Jon Hall5cf14d52015-07-16 12:15:19 -070097
Jon Hall4f360bc2017-09-07 10:19:52 -070098 main.HA.initialSetUp( serviceClean=True )
Jon Hall9d2dcad2016-04-08 10:15:20 -070099
Jon Hall5cf14d52015-07-16 12:15:19 -0700100 def CASE2( self, main ):
101 """
102 Assign devices to controllers
103 """
Devin Lim58046fa2017-07-05 16:55:00 -0700104 main.HA.assignDevices( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700105
106 def CASE21( self, main ):
107 """
108 Assign mastership to controllers
109 """
Devin Lim58046fa2017-07-05 16:55:00 -0700110 main.HA.assignMastership( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700111
112 def CASE3( self, main ):
113 """
114 Assign intents
115 """
Devin Lim58046fa2017-07-05 16:55:00 -0700116 main.HA.assignIntents( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700117
118 def CASE4( self, main ):
119 """
120 Ping across added host intents
121 """
Jon Hallca319892017-06-15 15:25:22 -0700122 main.HA.pingAcrossHostIntent( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700123
124 def CASE5( self, main ):
125 """
126 Reading state of ONOS
127 """
Devin Lim58046fa2017-07-05 16:55:00 -0700128 main.HA.readingState( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700129
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700130 def CASE61( self, main ):
Jon Hall5cf14d52015-07-16 12:15:19 -0700131 """
132 The Failure case.
133 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700134 assert main, "main not defined"
135 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700136 main.case( "Kill minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -0700137
138 main.step( "Checking ONOS Logs for errors" )
Jon Hallca319892017-06-15 15:25:22 -0700139 for ctrl in main.Cluster.active():
140 main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
141 main.log.warn( ctrl.checkLogs( ctrl.ipAddress ) )
Jon Hall96091e62015-09-21 17:34:17 -0700142
Devin Lim142b5342017-07-20 15:22:39 -0700143 n = len( main.Cluster.runningNodes ) # Number of nodes
Jon Hall3b489db2015-10-05 14:38:37 -0700144 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
Devin Lim142b5342017-07-20 15:22:39 -0700145 main.kill = [ main.Cluster.runningNodes[ 0 ] ] # ONOS node to kill, listed by index in main.nodes
Jon Hall3b489db2015-10-05 14:38:37 -0700146 if n > 3:
Devin Lim142b5342017-07-20 15:22:39 -0700147 main.kill.append( main.Cluster.runningNodes[ p - 1 ] )
Jon Hall3b489db2015-10-05 14:38:37 -0700148 # NOTE: This only works for cluster sizes of 3,5, or 7.
149
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700150 # NOTE: This is to fix an issue with wiki formating
Jon Hall4f360bc2017-09-07 10:19:52 -0700151 nodeNames = [ node.name for node in main.kill ]
152 main.step( "Killing nodes: " + str( nodeNames ) )
Jon Hall3b489db2015-10-05 14:38:37 -0700153 killResults = main.TRUE
Jon Hallca319892017-06-15 15:25:22 -0700154 for ctrl in main.kill:
Jon Hall3b489db2015-10-05 14:38:37 -0700155 killResults = killResults and\
Jon Hallca319892017-06-15 15:25:22 -0700156 ctrl.onosKill( ctrl.ipAddress )
157 ctrl.active = False
Jon Hall5cf14d52015-07-16 12:15:19 -0700158 utilities.assert_equals( expect=main.TRUE, actual=killResults,
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700159 onpass="ONOS nodes killed successfully",
160 onfail="ONOS nodes NOT successfully killed" )
161
Jon Halld2871c22016-07-26 11:01:14 -0700162 main.step( "Checking ONOS nodes" )
Devin Lim3ebd5e72017-11-14 10:38:00 -0800163 nodeResults = utilities.retry( main.Cluster.nodesCheck,
Jon Halld2871c22016-07-26 11:01:14 -0700164 False,
Jon Halld2871c22016-07-26 11:01:14 -0700165 sleep=15,
166 attempts=5 )
167
168 utilities.assert_equals( expect=True, actual=nodeResults,
169 onpass="Nodes check successful",
170 onfail="Nodes check NOT successful" )
171
172 if not nodeResults:
Jon Hallca319892017-06-15 15:25:22 -0700173 for ctrl in main.Cluster.active():
Jon Halld2871c22016-07-26 11:01:14 -0700174 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hallca319892017-06-15 15:25:22 -0700175 ctrl.name,
176 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Halld2871c22016-07-26 11:01:14 -0700177 main.log.error( "Failed to start ONOS, stopping test" )
Devin Lim44075962017-08-11 10:56:37 -0700178 main.cleanAndExit()
Jon Halld2871c22016-07-26 11:01:14 -0700179
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700180 def CASE62( self, main ):
181 """
182 The bring up stopped nodes
183 """
Jon Hall5d5876e2017-11-30 09:33:16 -0800184 main.HA.bringUpStoppedNodes( main )
Jon Halla440e872016-03-31 15:15:50 -0700185
Jon Hall5cf14d52015-07-16 12:15:19 -0700186 def CASE7( self, main ):
187 """
188 Check state after ONOS failure
189 """
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700190 try:
191 main.kill
192 except AttributeError:
193 main.kill = []
194
Devin Lim142b5342017-07-20 15:22:39 -0700195 main.HA.checkStateAfterEvent( main, afterWhich=0 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700196 main.step( "Leadership Election is still functional" )
197 # Test of LeadershipElection
198 leaderList = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700199
Jon Hall3b489db2015-10-05 14:38:37 -0700200 restarted = []
Jon Hallca319892017-06-15 15:25:22 -0700201 for ctrl in main.kill:
202 restarted.append( ctrl.ipAddress )
Jon Hall5cf14d52015-07-16 12:15:19 -0700203 leaderResult = main.TRUE
Jon Hall3b489db2015-10-05 14:38:37 -0700204
Jon Hallca319892017-06-15 15:25:22 -0700205 for ctrl in main.Cluster.active():
206 leaderN = ctrl.electionTestLeader()
Jon Hall5cf14d52015-07-16 12:15:19 -0700207 leaderList.append( leaderN )
208 if leaderN == main.FALSE:
209 # error in response
210 main.log.error( "Something is wrong with " +
211 "electionTestLeader function, check the" +
212 " error logs" )
213 leaderResult = main.FALSE
214 elif leaderN is None:
Jon Hallca319892017-06-15 15:25:22 -0700215 main.log.error( ctrl.name +
Jon Hall5cf14d52015-07-16 12:15:19 -0700216 " shows no leader for the election-app was" +
217 " elected after the old one died" )
218 leaderResult = main.FALSE
219 elif leaderN in restarted:
Jon Hallca319892017-06-15 15:25:22 -0700220 main.log.error( ctrl.name + " shows " + str( leaderN ) +
Jon Hall5cf14d52015-07-16 12:15:19 -0700221 " as leader for the election-app, but it " +
222 "was restarted" )
223 leaderResult = main.FALSE
224 if len( set( leaderList ) ) != 1:
225 leaderResult = main.FALSE
226 main.log.error(
227 "Inconsistent view of leader for the election test app" )
228 # TODO: print the list
229 utilities.assert_equals(
230 expect=main.TRUE,
231 actual=leaderResult,
232 onpass="Leadership election passed",
233 onfail="Something went wrong with Leadership election" )
234
235 def CASE8( self, main ):
236 """
237 Compare topo
238 """
Devin Lim58046fa2017-07-05 16:55:00 -0700239 main.HA.compareTopo( main )
Jon Halld2871c22016-07-26 11:01:14 -0700240
Jon Hall5cf14d52015-07-16 12:15:19 -0700241 def CASE9( self, main ):
242 """
243 Link s3-s28 down
244 """
Devin Lim58046fa2017-07-05 16:55:00 -0700245 main.HA.linkDown( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700246
247 def CASE10( self, main ):
248 """
249 Link s3-s28 up
250 """
Devin Lim58046fa2017-07-05 16:55:00 -0700251 main.HA.linkUp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700252
253 def CASE11( self, main ):
254 """
255 Switch Down
256 """
257 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700258 main.HA.switchDown( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700259
260 def CASE12( self, main ):
261 """
262 Switch Up
263 """
264 # NOTE: You should probably run a topology check after this
Devin Lim58046fa2017-07-05 16:55:00 -0700265 main.HA.switchUp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700266
267 def CASE13( self, main ):
268 """
269 Clean up
270 """
Devin Lim58046fa2017-07-05 16:55:00 -0700271 main.HAlabels.append( "Restart" )
272 main.HAdata.append( str( main.restartTime ) )
273 main.HA.cleanUp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700274
275 def CASE14( self, main ):
276 """
277 start election app on all onos nodes
278 """
Devin Lim58046fa2017-07-05 16:55:00 -0700279 main.HA.startElectionApp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700280
281 def CASE15( self, main ):
282 """
283 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -0700284 15.1 Run election on each node
285 15.2 Check that each node has the same leaders and candidates
286 15.3 Find current leader and withdraw
287 15.4 Check that a new node was elected leader
288 15.5 Check that that new leader was the candidate of old leader
289 15.6 Run for election on old leader
290 15.7 Check that oldLeader is a candidate, and leader if only 1 node
291 15.8 Make sure that the old leader was added to the candidate list
292
293 old and new variable prefixes refer to data from before vs after
294 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -0700295 """
Devin Lim58046fa2017-07-05 16:55:00 -0700296 main.HA.isElectionFunctional( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700297
298 def CASE16( self, main ):
299 """
300 Install Distributed Primitives app
301 """
Devin Lim58046fa2017-07-05 16:55:00 -0700302 main.HA.installDistributedPrimitiveApp( main )
Jon Hall5cf14d52015-07-16 12:15:19 -0700303
304 def CASE17( self, main ):
305 """
306 Check for basic functionality with distributed primitives
307 """
Devin Lim58046fa2017-07-05 16:55:00 -0700308 main.HA.checkDistPrimitivesFunc( main )