| Pier | a2a7e1b | 2016-10-04 11:51:43 -0700 | [diff] [blame] | 1 | # In this test we perform several failures and then test for connectivity |
| 2 | # CASE1: 2x2 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures |
| 3 | # CASE2: 2x2 topo + 3 ONOS + | ONOS (random instance) failure + IP connectivity test | x failures |
| 4 | # CASE3: 4x4 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures |
| 5 | # CASE4: 4x4 topo + 3 ONOS + | ONOS (random instance) failure + IP connectivity test | x failures |
| 6 | # CASE5: 2x2 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures |
| 7 | # CASE6: 2x2 topo + 3 ONOS + | ONOS (random instance) failure + Spine (random switch) failure + IP connectivity test | x failures |
| 8 | # CASE7: 4x4 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures |
| 9 | # CASE8: 4x4 topo + 3 ONOS + | ONOS (random instance) failure + Spine (random switch) failure + IP connectivity test | x failures |
| 10 | |
| 11 | |
| 12 | |
| 13 | class SRHighAvailability: |
| 14 | |
| 15 | def __init__( self ): |
| 16 | self.default = '' |
| 17 | |
| 18 | def CASE1( self, main ): |
| 19 | """ |
| 20 | 1) Sets up 3-nodes Onos-cluster |
| 21 | 2) Start 2x2 Leaf-Spine topology |
| 22 | 3) Pingall |
| 23 | 4) Cause sequential ONOS failure |
| 24 | 5) Pingall |
| 25 | 6) Repeat 3), 4), 5) 'failures' times |
| 26 | """ |
| 27 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \ |
| 28 | Testcaselib as run |
| 29 | if not hasattr( main, 'apps' ): |
| 30 | run.initTest( main ) |
| 31 | |
| 32 | description = "High Availability tests - ONOS failures with 2x2 Leaf-spine " |
| 33 | main.case( description ) |
| 34 | run.config(main, '2x2', 3) |
| 35 | run.installOnos( main ) |
| 36 | run.startMininet( main, 'cord_fabric.py' ) |
| 37 | # pre-configured routing and bridging test |
| 38 | run.checkFlows( main, minFlowCount=116 ) |
| 39 | run.pingAll( main ) |
| 40 | for i in range(0, main.failures): |
| 41 | toKill = i % main.numCtrls |
| 42 | run.killOnos( main, [ toKill ], '4', '8', '2' ) |
| 43 | run.pingAll( main, 'CASE1_Failure%d' % (i+1) ) |
| 44 | run.recoverOnos( main, [ toKill ], '4', '8', '3' ) |
| 45 | run.checkFlows( main, minFlowCount=116 ) |
| 46 | run.pingAll( main, 'CASE1_Recovery%d' % (i+1) ) |
| 47 | run.cleanup( main ) |
| 48 | |
| 49 | def CASE2( self, main ): |
| 50 | """ |
| 51 | 1) Sets up 3-nodes Onos-cluster |
| 52 | 2) Start 2x2 Leaf-Spine topology |
| 53 | 3) Pingall |
| 54 | 4) Cause random ONOS failure |
| 55 | 5) Pingall |
| 56 | 6) Repeat 3), 4), 5) 'failures' times |
| 57 | """ |
| 58 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \ |
| 59 | Testcaselib as run |
| 60 | import random |
| 61 | from random import randint |
| 62 | from datetime import datetime |
| 63 | if not hasattr( main, 'apps' ): |
| 64 | run.initTest( main ) |
| 65 | |
| 66 | description = "High Availability tests - ONOS random failures with 2x2 Leaf-spine " |
| 67 | main.case( description ) |
| 68 | run.config(main, '2x2', 3) |
| 69 | run.installOnos( main ) |
| 70 | run.startMininet( main, 'cord_fabric.py' ) |
| 71 | # pre-configured routing and bridging test |
| 72 | run.checkFlows( main, minFlowCount=116 ) |
| 73 | run.pingAll( main ) |
| 74 | random.seed(datetime.now()) |
| 75 | for i in range(0, main.failures): |
| 76 | toKill = randint(0, (main.numCtrls-1)) |
| 77 | run.killOnos( main, [ toKill ], '4', '8', '2' ) |
| 78 | run.pingAll( main, 'CASE2_Failure%d' % (i+1) ) |
| 79 | run.recoverOnos( main, [ toKill ], '4', '8', '3' ) |
| 80 | run.checkFlows( main, minFlowCount=116 ) |
| 81 | run.pingAll( main, 'CASE2_Recovery%d' % (i+1) ) |
| 82 | run.cleanup( main ) |
| 83 | |
| 84 | def CASE3( self, main ): |
| 85 | """ |
| 86 | 1) Sets up 3-nodes Onos-cluster |
| 87 | 2) Start 4x4 Leaf-Spine topology |
| 88 | 3) Pingall |
| 89 | 4) Cause sequential ONOS failure |
| 90 | 5) Pingall |
| 91 | 6) Repeat 3), 4), 5) 'failures' times |
| 92 | """ |
| 93 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \ |
| 94 | Testcaselib as run |
| 95 | if not hasattr( main, 'apps' ): |
| 96 | run.initTest( main ) |
| 97 | |
| 98 | description = "High Availability tests - ONOS failures with 4x4 Leaf-spine " |
| 99 | main.case( description ) |
| 100 | run.config(main, '4x4', 3) |
| 101 | run.installOnos( main ) |
| 102 | run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" ) |
| 103 | # pre-configured routing and bridging test |
| 104 | run.checkFlows( main, minFlowCount=350 ) |
| 105 | run.pingAll( main ) |
| 106 | for i in range(0, main.failures): |
| 107 | toKill = i % main.numCtrls |
| 108 | run.killOnos( main, [ toKill ], '8', '32', '2' ) |
| 109 | run.pingAll( main, 'CASE3_Failure%d' % (i+1) ) |
| 110 | run.recoverOnos( main, [ toKill ], '8', '32', '3' ) |
| 111 | run.checkFlows( main, minFlowCount=350 ) |
| 112 | run.pingAll( main, 'CASE3_Recovery%d' % (i+1) ) |
| 113 | run.cleanup( main ) |
| 114 | |
| 115 | def CASE4( self, main ): |
| 116 | """ |
| 117 | 1) Sets up 3-nodes Onos-cluster |
| 118 | 2) Start 4x4 Leaf-Spine topology |
| 119 | 3) Pingall |
| 120 | 4) Cause random ONOS failure |
| 121 | 5) Pingall |
| 122 | 6) Repeat 3), 4), 5) 'failures' times |
| 123 | """ |
| 124 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \ |
| 125 | Testcaselib as run |
| 126 | import random |
| 127 | from random import randint |
| 128 | from datetime import datetime |
| 129 | if not hasattr( main, 'apps' ): |
| 130 | run.initTest( main ) |
| 131 | |
| 132 | description = "High Availability tests - ONOS random failures with 4x4 Leaf-spine " |
| 133 | main.case( description ) |
| 134 | run.config(main, '4x4', 3) |
| 135 | run.installOnos( main ) |
| 136 | run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" ) |
| 137 | # pre-configured routing and bridging test |
| 138 | run.checkFlows( main, minFlowCount=350 ) |
| 139 | run.pingAll( main ) |
| 140 | random.seed(datetime.now()) |
| 141 | for i in range(0, main.failures): |
| 142 | toKill = randint(0, (main.numCtrls-1)) |
| 143 | run.killOnos( main, [ toKill ], '8', '32', '2' ) |
| 144 | run.pingAll( main, 'CASE4_Failure%d' % (i+1) ) |
| 145 | run.recoverOnos( main, [ toKill ], '8', '32', '3' ) |
| 146 | run.checkFlows( main, minFlowCount=350 ) |
| 147 | run.pingAll( main, 'CASE4_Recovery%d' % (i+1) ) |
| 148 | run.cleanup( main ) |
| 149 | |
| 150 | def CASE5( self, main ): |
| 151 | """ |
| 152 | 1) Sets up 3-nodes Onos-cluster |
| 153 | 2) Start 2x2 Leaf-Spine topology |
| 154 | 3) Pingall |
| 155 | 4) Cause sequential ONOS failure |
| 156 | 5) Pingall |
| 157 | 6) Cause sequential Spine failure |
| 158 | 7) Pingall |
| 159 | 8) Repeat 3), 4), 5), 6), 7), 'failures' times |
| 160 | """ |
| 161 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \ |
| 162 | Testcaselib as run |
| 163 | import time |
| 164 | if not hasattr( main, 'apps' ): |
| 165 | run.initTest( main ) |
| 166 | |
| 167 | description = "High Availability tests - ONOS failures and Switch failures with 2x2 Leaf-spine " |
| 168 | main.case( description ) |
| 169 | run.config(main, '2x2', 3) |
| 170 | run.installOnos( main ) |
| 171 | run.startMininet( main, 'cord_fabric.py' ) |
| 172 | # pre-configured routing and bridging test |
| 173 | run.checkFlows( main, minFlowCount=116 ) |
| 174 | run.pingAll( main ) |
| 175 | for i in range(0, main.failures): |
| 176 | onosToKill = i % main.numCtrls |
| 177 | switchToKill = i % len(main.spines) |
| 178 | run.killOnos( main, [ onosToKill ], '4', '8', '2' ) |
| 179 | run.pingAll( main, 'CASE5_ONOS_Failure%d' % (i+1) ) |
| 180 | run.killSwitch( main, main.spines[switchToKill]['name'], switches='3', links='4' ) |
| 181 | time.sleep( main.switchSleep ) |
| 182 | run.pingAll( main, "CASE5_SWITCH_Failure%d" % (i+1) ) |
| 183 | run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='4', links='8' ) |
| 184 | run.checkFlows( main, minFlowCount=116 ) |
| 185 | run.pingAll( main, "CASE5_SWITCH_Recovery%d" % (i+1) ) |
| 186 | run.recoverOnos( main, [ onosToKill ], '4', '8', '3' ) |
| 187 | run.checkFlows( main, minFlowCount=116 ) |
| 188 | run.pingAll( main, 'CASE5_ONOS_Recovery%d' % (i+1) ) |
| 189 | run.cleanup( main ) |
| 190 | |
| 191 | def CASE6( self, main ): |
| 192 | """ |
| 193 | 1) Sets up 3-nodes Onos-cluster |
| 194 | 2) Start 2x2 Leaf-Spine topology |
| 195 | 3) Pingall |
| 196 | 4) Cause random ONOS failure |
| 197 | 5) Pingall |
| 198 | 6) Cause random Spine failure |
| 199 | 7) Pingall |
| 200 | 8) Repeat 3), 4), 5), 6), 7) 'failures' times |
| 201 | """ |
| 202 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \ |
| 203 | Testcaselib as run |
| 204 | import time |
| 205 | import random |
| 206 | from random import randint |
| 207 | from datetime import datetime |
| 208 | if not hasattr( main, 'apps' ): |
| 209 | run.initTest( main ) |
| 210 | |
| 211 | description = "High Availability tests - ONOS random failures and Switch random failures with 2x2 Leaf-spine " |
| 212 | main.case( description ) |
| 213 | run.config(main, '2x2', 3) |
| 214 | run.installOnos( main ) |
| 215 | run.startMininet( main, 'cord_fabric.py' ) |
| 216 | # pre-configured routing and bridging test |
| 217 | run.checkFlows( main, minFlowCount=116 ) |
| 218 | run.pingAll( main ) |
| 219 | for i in range(0, main.failures): |
| 220 | onosToKill = randint(0, (main.numCtrls-1)) |
| 221 | switchToKill = randint(0, 1) |
| 222 | run.killOnos( main, [ onosToKill ], '4', '8', '2' ) |
| 223 | run.pingAll( main, 'CASE6_ONOS_Failure%d' % (i+1) ) |
| 224 | run.killSwitch( main, main.spines[switchToKill]['name'], switches='3', links='4' ) |
| 225 | time.sleep( main.switchSleep ) |
| 226 | run.pingAll( main, "CASE6_SWITCH_Failure%d" % (i+1) ) |
| 227 | run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='4', links='8' ) |
| 228 | run.checkFlows( main, minFlowCount=116 ) |
| 229 | run.pingAll( main, "CASE6_SWITCH_Recovery%d" % (i+1) ) |
| 230 | run.recoverOnos( main, [ onosToKill ], '4', '8', '3' ) |
| 231 | run.checkFlows( main, minFlowCount=116 ) |
| 232 | run.pingAll( main, 'CASE6_ONOS_Recovery%d' % (i+1) ) |
| 233 | run.cleanup( main ) |
| 234 | |
| 235 | def CASE7( self, main ): |
| 236 | """ |
| 237 | 1) Sets up 3-nodes Onos-cluster |
| 238 | 2) Start 4x4 Leaf-Spine topology |
| 239 | 3) Pingall |
| 240 | 4) Cause sequential ONOS failure |
| 241 | 5) Pingall |
| 242 | 6) Cause sequential Spine failure |
| 243 | 7) Pingall |
| 244 | 8) Repeat 3), 4), 5), 6), 7), 'failures' times |
| 245 | """ |
| 246 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \ |
| 247 | Testcaselib as run |
| 248 | import time |
| 249 | if not hasattr( main, 'apps' ): |
| 250 | run.initTest( main ) |
| 251 | |
| 252 | description = "High Availability tests - ONOS failures and Switch failures with 4x4 Leaf-spine " |
| 253 | main.case( description ) |
| 254 | run.config(main, '4x4', 3) |
| 255 | run.installOnos( main ) |
| 256 | run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" ) |
| 257 | # pre-configured routing and bridging test |
| 258 | run.checkFlows( main, minFlowCount=350 ) |
| 259 | run.pingAll( main ) |
| 260 | for i in range(0, main.failures): |
| 261 | onosToKill = i % main.numCtrls |
| 262 | switchToKill = i % len(main.spines) |
| 263 | run.killOnos( main, [ onosToKill ], '8', '32', '2' ) |
| 264 | run.pingAll( main, 'CASE7_ONOS_Failure%d' % (i+1) ) |
| 265 | run.killSwitch( main, main.spines[switchToKill]['name'], switches='7', links='24' ) |
| 266 | time.sleep( main.switchSleep ) |
| 267 | run.pingAll( main, "CASE7_SWITCH_Failure%d" % (i+1) ) |
| 268 | run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='8', links='32' ) |
| 269 | run.checkFlows( main, minFlowCount=350 ) |
| 270 | run.pingAll( main, "CASE7_SWITCH_Recovery%d" % (i+1) ) |
| 271 | run.recoverOnos( main, [ onosToKill ], '8', '32', '3' ) |
| 272 | run.checkFlows( main, minFlowCount=350 ) |
| 273 | run.pingAll( main, 'CASE7_ONOS_Recovery%d' % (i+1) ) |
| 274 | run.cleanup( main ) |
| 275 | |
| 276 | def CASE8( self, main ): |
| 277 | """ |
| 278 | 1) Sets up 3-nodes Onos-cluster |
| 279 | 2) Start 4x4 Leaf-Spine topology |
| 280 | 3) Pingall |
| 281 | 4) Cause random ONOS failure |
| 282 | 5) Pingall |
| 283 | 6) Cause random Spine failure |
| 284 | 7) Pingall |
| 285 | 8) Repeat 3), 4), 5), 6), 7), 'failures' times |
| 286 | """ |
| 287 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \ |
| 288 | Testcaselib as run |
| 289 | import time |
| 290 | import random |
| 291 | from random import randint |
| 292 | from datetime import datetime |
| 293 | if not hasattr( main, 'apps' ): |
| 294 | run.initTest( main ) |
| 295 | |
| 296 | description = "High Availability tests - ONOS random failures and Switch random failures with 4x4 Leaf-spine " |
| 297 | main.case( description ) |
| 298 | run.config(main, '4x4', 3) |
| 299 | run.installOnos( main ) |
| 300 | run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" ) |
| 301 | # pre-configured routing and bridging test |
| 302 | run.checkFlows( main, minFlowCount=350 ) |
| 303 | run.pingAll( main ) |
| 304 | for i in range(0, main.failures): |
| 305 | onosToKill = randint(0, (main.numCtrls-1)) |
| 306 | switchToKill = randint(0, 3) |
| 307 | run.killOnos( main, [ onosToKill ], '8', '32', '2' ) |
| 308 | run.pingAll( main, 'CASE8_ONOS_Failure%d' % (i+1) ) |
| 309 | run.killSwitch( main, main.spines[switchToKill]['name'], switches='7', links='24' ) |
| 310 | time.sleep( main.switchSleep ) |
| 311 | run.pingAll( main, "CASE8_SWITCH_Failure%d" % (i+1) ) |
| 312 | run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='8', links='32' ) |
| 313 | run.checkFlows( main, minFlowCount=350 ) |
| 314 | run.pingAll( main, "CASE8_SWITCH_Recovery%d" % (i+1) ) |
| 315 | run.recoverOnos( main, [ onosToKill ], '8', '32', '3' ) |
| 316 | run.checkFlows( main, minFlowCount=350 ) |
| 317 | run.pingAll( main, 'CASE8_ONOS_Recovery%d' % (i+1) ) |
| 318 | run.cleanup( main ) |