blob: ba0242db1b2387952f9755e15e59181d7a870c6c [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
2Copyright 2016 Open Networking Foundation (ONF)
3
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
11 (at your option) any later version.
12
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
21
Piera2a7e1b2016-10-04 11:51:43 -070022# In this test we perform several failures and then test for connectivity
23# CASE1: 2x2 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures
24# CASE2: 2x2 topo + 3 ONOS + | ONOS (random instance) failure + IP connectivity test | x failures
25# CASE3: 4x4 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures
26# CASE4: 4x4 topo + 3 ONOS + | ONOS (random instance) failure + IP connectivity test | x failures
27# CASE5: 2x2 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures
28# CASE6: 2x2 topo + 3 ONOS + | ONOS (random instance) failure + Spine (random switch) failure + IP connectivity test | x failures
29# CASE7: 4x4 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures
30# CASE8: 4x4 topo + 3 ONOS + | ONOS (random instance) failure + Spine (random switch) failure + IP connectivity test | x failures
31
32
33
34class SRHighAvailability:
35
36 def __init__( self ):
37 self.default = ''
38
39 def CASE1( self, main ):
40 """
41 1) Sets up 3-nodes Onos-cluster
42 2) Start 2x2 Leaf-Spine topology
43 3) Pingall
44 4) Cause sequential ONOS failure
45 5) Pingall
46 6) Repeat 3), 4), 5) 'failures' times
47 """
48 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
49 Testcaselib as run
50 if not hasattr( main, 'apps' ):
51 run.initTest( main )
52
53 description = "High Availability tests - ONOS failures with 2x2 Leaf-spine "
54 main.case( description )
Devin Lim142b5342017-07-20 15:22:39 -070055 run.config(main, '2x2')
Piera2a7e1b2016-10-04 11:51:43 -070056 run.installOnos( main )
57 run.startMininet( main, 'cord_fabric.py' )
58 # pre-configured routing and bridging test
59 run.checkFlows( main, minFlowCount=116 )
60 run.pingAll( main )
61 for i in range(0, main.failures):
Devin Lim142b5342017-07-20 15:22:39 -070062 toKill = i % main.Cluster.numCtrls
Piera2a7e1b2016-10-04 11:51:43 -070063 run.killOnos( main, [ toKill ], '4', '8', '2' )
64 run.pingAll( main, 'CASE1_Failure%d' % (i+1) )
65 run.recoverOnos( main, [ toKill ], '4', '8', '3' )
66 run.checkFlows( main, minFlowCount=116 )
67 run.pingAll( main, 'CASE1_Recovery%d' % (i+1) )
68 run.cleanup( main )
69
70 def CASE2( self, main ):
71 """
72 1) Sets up 3-nodes Onos-cluster
73 2) Start 2x2 Leaf-Spine topology
74 3) Pingall
75 4) Cause random ONOS failure
76 5) Pingall
77 6) Repeat 3), 4), 5) 'failures' times
78 """
79 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
80 Testcaselib as run
81 import random
82 from random import randint
83 from datetime import datetime
84 if not hasattr( main, 'apps' ):
85 run.initTest( main )
86
87 description = "High Availability tests - ONOS random failures with 2x2 Leaf-spine "
88 main.case( description )
Devin Lim142b5342017-07-20 15:22:39 -070089 run.config(main, '2x2')
Piera2a7e1b2016-10-04 11:51:43 -070090 run.installOnos( main )
91 run.startMininet( main, 'cord_fabric.py' )
92 # pre-configured routing and bridging test
93 run.checkFlows( main, minFlowCount=116 )
94 run.pingAll( main )
95 random.seed(datetime.now())
96 for i in range(0, main.failures):
Devin Lim142b5342017-07-20 15:22:39 -070097 toKill = randint( 0, ( main.Cluster.numCtrls - 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -070098 run.killOnos( main, [ toKill ], '4', '8', '2' )
99 run.pingAll( main, 'CASE2_Failure%d' % (i+1) )
100 run.recoverOnos( main, [ toKill ], '4', '8', '3' )
101 run.checkFlows( main, minFlowCount=116 )
102 run.pingAll( main, 'CASE2_Recovery%d' % (i+1) )
103 run.cleanup( main )
104
105 def CASE3( self, main ):
106 """
107 1) Sets up 3-nodes Onos-cluster
108 2) Start 4x4 Leaf-Spine topology
109 3) Pingall
110 4) Cause sequential ONOS failure
111 5) Pingall
112 6) Repeat 3), 4), 5) 'failures' times
113 """
114 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
115 Testcaselib as run
116 if not hasattr( main, 'apps' ):
117 run.initTest( main )
118
119 description = "High Availability tests - ONOS failures with 4x4 Leaf-spine "
120 main.case( description )
Devin Lim142b5342017-07-20 15:22:39 -0700121 run.config(main, '4x4')
Piera2a7e1b2016-10-04 11:51:43 -0700122 run.installOnos( main )
123 run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
124 # pre-configured routing and bridging test
125 run.checkFlows( main, minFlowCount=350 )
126 run.pingAll( main )
127 for i in range(0, main.failures):
Devin Lim142b5342017-07-20 15:22:39 -0700128 toKill = i % main.Cluster.numCtrls
Piera2a7e1b2016-10-04 11:51:43 -0700129 run.killOnos( main, [ toKill ], '8', '32', '2' )
130 run.pingAll( main, 'CASE3_Failure%d' % (i+1) )
131 run.recoverOnos( main, [ toKill ], '8', '32', '3' )
132 run.checkFlows( main, minFlowCount=350 )
133 run.pingAll( main, 'CASE3_Recovery%d' % (i+1) )
134 run.cleanup( main )
135
136 def CASE4( self, main ):
137 """
138 1) Sets up 3-nodes Onos-cluster
139 2) Start 4x4 Leaf-Spine topology
140 3) Pingall
141 4) Cause random ONOS failure
142 5) Pingall
143 6) Repeat 3), 4), 5) 'failures' times
144 """
145 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
146 Testcaselib as run
147 import random
148 from random import randint
149 from datetime import datetime
150 if not hasattr( main, 'apps' ):
151 run.initTest( main )
152
153 description = "High Availability tests - ONOS random failures with 4x4 Leaf-spine "
154 main.case( description )
Devin Lim142b5342017-07-20 15:22:39 -0700155 run.config(main, '4x4')
Piera2a7e1b2016-10-04 11:51:43 -0700156 run.installOnos( main )
157 run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
158 # pre-configured routing and bridging test
159 run.checkFlows( main, minFlowCount=350 )
160 run.pingAll( main )
161 random.seed(datetime.now())
162 for i in range(0, main.failures):
Devin Lim142b5342017-07-20 15:22:39 -0700163 toKill = randint( 0, ( main.Cluster.numCtrls - 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700164 run.killOnos( main, [ toKill ], '8', '32', '2' )
165 run.pingAll( main, 'CASE4_Failure%d' % (i+1) )
166 run.recoverOnos( main, [ toKill ], '8', '32', '3' )
167 run.checkFlows( main, minFlowCount=350 )
168 run.pingAll( main, 'CASE4_Recovery%d' % (i+1) )
169 run.cleanup( main )
170
171 def CASE5( self, main ):
172 """
173 1) Sets up 3-nodes Onos-cluster
174 2) Start 2x2 Leaf-Spine topology
175 3) Pingall
176 4) Cause sequential ONOS failure
177 5) Pingall
178 6) Cause sequential Spine failure
179 7) Pingall
180 8) Repeat 3), 4), 5), 6), 7), 'failures' times
181 """
182 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
183 Testcaselib as run
184 import time
185 if not hasattr( main, 'apps' ):
186 run.initTest( main )
187
188 description = "High Availability tests - ONOS failures and Switch failures with 2x2 Leaf-spine "
189 main.case( description )
Devin Lim142b5342017-07-20 15:22:39 -0700190 run.config(main, '2x2')
Piera2a7e1b2016-10-04 11:51:43 -0700191 run.installOnos( main )
192 run.startMininet( main, 'cord_fabric.py' )
193 # pre-configured routing and bridging test
194 run.checkFlows( main, minFlowCount=116 )
195 run.pingAll( main )
196 for i in range(0, main.failures):
Devin Lim142b5342017-07-20 15:22:39 -0700197 onosToKill = i % main.Cluster.numCtrls
Piera2a7e1b2016-10-04 11:51:43 -0700198 switchToKill = i % len(main.spines)
199 run.killOnos( main, [ onosToKill ], '4', '8', '2' )
200 run.pingAll( main, 'CASE5_ONOS_Failure%d' % (i+1) )
201 run.killSwitch( main, main.spines[switchToKill]['name'], switches='3', links='4' )
202 time.sleep( main.switchSleep )
203 run.pingAll( main, "CASE5_SWITCH_Failure%d" % (i+1) )
204 run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='4', links='8' )
205 run.checkFlows( main, minFlowCount=116 )
206 run.pingAll( main, "CASE5_SWITCH_Recovery%d" % (i+1) )
207 run.recoverOnos( main, [ onosToKill ], '4', '8', '3' )
208 run.checkFlows( main, minFlowCount=116 )
209 run.pingAll( main, 'CASE5_ONOS_Recovery%d' % (i+1) )
210 run.cleanup( main )
211
212 def CASE6( self, main ):
213 """
214 1) Sets up 3-nodes Onos-cluster
215 2) Start 2x2 Leaf-Spine topology
216 3) Pingall
217 4) Cause random ONOS failure
218 5) Pingall
219 6) Cause random Spine failure
220 7) Pingall
221 8) Repeat 3), 4), 5), 6), 7) 'failures' times
222 """
223 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
224 Testcaselib as run
225 import time
226 import random
227 from random import randint
228 from datetime import datetime
229 if not hasattr( main, 'apps' ):
230 run.initTest( main )
231
232 description = "High Availability tests - ONOS random failures and Switch random failures with 2x2 Leaf-spine "
233 main.case( description )
Devin Lim142b5342017-07-20 15:22:39 -0700234 run.config(main, '2x2')
Piera2a7e1b2016-10-04 11:51:43 -0700235 run.installOnos( main )
236 run.startMininet( main, 'cord_fabric.py' )
237 # pre-configured routing and bridging test
238 run.checkFlows( main, minFlowCount=116 )
239 run.pingAll( main )
240 for i in range(0, main.failures):
Devin Lim142b5342017-07-20 15:22:39 -0700241 onosToKill = randint( 0, ( main.Cluster.numCtrls - 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700242 switchToKill = randint(0, 1)
243 run.killOnos( main, [ onosToKill ], '4', '8', '2' )
244 run.pingAll( main, 'CASE6_ONOS_Failure%d' % (i+1) )
245 run.killSwitch( main, main.spines[switchToKill]['name'], switches='3', links='4' )
246 time.sleep( main.switchSleep )
247 run.pingAll( main, "CASE6_SWITCH_Failure%d" % (i+1) )
248 run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='4', links='8' )
249 run.checkFlows( main, minFlowCount=116 )
250 run.pingAll( main, "CASE6_SWITCH_Recovery%d" % (i+1) )
251 run.recoverOnos( main, [ onosToKill ], '4', '8', '3' )
252 run.checkFlows( main, minFlowCount=116 )
253 run.pingAll( main, 'CASE6_ONOS_Recovery%d' % (i+1) )
254 run.cleanup( main )
255
256 def CASE7( self, main ):
257 """
258 1) Sets up 3-nodes Onos-cluster
259 2) Start 4x4 Leaf-Spine topology
260 3) Pingall
261 4) Cause sequential ONOS failure
262 5) Pingall
263 6) Cause sequential Spine failure
264 7) Pingall
265 8) Repeat 3), 4), 5), 6), 7), 'failures' times
266 """
267 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
268 Testcaselib as run
269 import time
270 if not hasattr( main, 'apps' ):
271 run.initTest( main )
272
273 description = "High Availability tests - ONOS failures and Switch failures with 4x4 Leaf-spine "
274 main.case( description )
Devin Lim142b5342017-07-20 15:22:39 -0700275 run.config(main, '4x4')
Piera2a7e1b2016-10-04 11:51:43 -0700276 run.installOnos( main )
277 run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
278 # pre-configured routing and bridging test
279 run.checkFlows( main, minFlowCount=350 )
280 run.pingAll( main )
281 for i in range(0, main.failures):
Devin Lim142b5342017-07-20 15:22:39 -0700282 onosToKill = i % main.Cluster.numCtrls
Piera2a7e1b2016-10-04 11:51:43 -0700283 switchToKill = i % len(main.spines)
284 run.killOnos( main, [ onosToKill ], '8', '32', '2' )
285 run.pingAll( main, 'CASE7_ONOS_Failure%d' % (i+1) )
286 run.killSwitch( main, main.spines[switchToKill]['name'], switches='7', links='24' )
287 time.sleep( main.switchSleep )
288 run.pingAll( main, "CASE7_SWITCH_Failure%d" % (i+1) )
289 run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='8', links='32' )
290 run.checkFlows( main, minFlowCount=350 )
291 run.pingAll( main, "CASE7_SWITCH_Recovery%d" % (i+1) )
292 run.recoverOnos( main, [ onosToKill ], '8', '32', '3' )
293 run.checkFlows( main, minFlowCount=350 )
294 run.pingAll( main, 'CASE7_ONOS_Recovery%d' % (i+1) )
295 run.cleanup( main )
296
297 def CASE8( self, main ):
298 """
299 1) Sets up 3-nodes Onos-cluster
300 2) Start 4x4 Leaf-Spine topology
301 3) Pingall
302 4) Cause random ONOS failure
303 5) Pingall
304 6) Cause random Spine failure
305 7) Pingall
306 8) Repeat 3), 4), 5), 6), 7), 'failures' times
307 """
308 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
309 Testcaselib as run
310 import time
311 import random
312 from random import randint
313 from datetime import datetime
314 if not hasattr( main, 'apps' ):
315 run.initTest( main )
316
317 description = "High Availability tests - ONOS random failures and Switch random failures with 4x4 Leaf-spine "
318 main.case( description )
Devin Lim142b5342017-07-20 15:22:39 -0700319 run.config(main, '4x4')
Piera2a7e1b2016-10-04 11:51:43 -0700320 run.installOnos( main )
321 run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
322 # pre-configured routing and bridging test
323 run.checkFlows( main, minFlowCount=350 )
324 run.pingAll( main )
325 for i in range(0, main.failures):
Devin Lim142b5342017-07-20 15:22:39 -0700326 onosToKill = randint( 0, ( main.Cluster.numCtrls - 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700327 switchToKill = randint(0, 3)
328 run.killOnos( main, [ onosToKill ], '8', '32', '2' )
329 run.pingAll( main, 'CASE8_ONOS_Failure%d' % (i+1) )
330 run.killSwitch( main, main.spines[switchToKill]['name'], switches='7', links='24' )
331 time.sleep( main.switchSleep )
332 run.pingAll( main, "CASE8_SWITCH_Failure%d" % (i+1) )
333 run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='8', links='32' )
334 run.checkFlows( main, minFlowCount=350 )
335 run.pingAll( main, "CASE8_SWITCH_Recovery%d" % (i+1) )
336 run.recoverOnos( main, [ onosToKill ], '8', '32', '3' )
337 run.checkFlows( main, minFlowCount=350 )
338 run.pingAll( main, 'CASE8_ONOS_Recovery%d' % (i+1) )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -0700339 run.cleanup( main )