Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 1 | """ |
Jeremy Ronquillo | b27ce4c | 2017-07-17 12:41:28 -0700 | [diff] [blame] | 2 | Copyright 2017 Open Networking Foundation (ONF) |
| 3 | |
| 4 | Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>, |
| 5 | the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>, |
| 6 | or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg> |
| 7 | |
| 8 | TestON is free software: you can redistribute it and/or modify |
| 9 | it under the terms of the GNU General Public License as published by |
| 10 | the Free Software Foundation, either version 2 of the License, or |
| 11 | (at your option) any later version. |
| 12 | |
| 13 | TestON is distributed in the hope that it will be useful, |
| 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | GNU General Public License for more details. |
| 17 | |
| 18 | You should have received a copy of the GNU General Public License |
| 19 | along with TestON. If not, see <http://www.gnu.org/licenses/>. |
| 20 | """ |
| 21 | |
| 22 | """ |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 23 | Testing various connectivity failures for VPLS. |
| 24 | |
| 25 | CASE1: Startup. |
| 26 | CASE2: Load VPLS topology and configurations from demo script. |
| 27 | CASE50: Initial connectivity test. |
| 28 | CASE100: Bring down 1 host at a time and test connectivity. |
| 29 | CASE200: Bring down 1 switch at a time and test connectivity. |
| 30 | CASE300: Stop 1 ONOS node at a time and test connectivity. |
| 31 | CASE310: Kill 1 ONOS node at a time and test connectivity. |
| 32 | CASE400: Bring down 1 link at a time and test connectivity. |
| 33 | """ |
| 34 | |
| 35 | class VPLSfailsafe: |
| 36 | |
| 37 | def __init__( self ): |
| 38 | self.default = '' |
| 39 | |
| 40 | |
| 41 | def CASE1( self, main ): |
| 42 | """ |
| 43 | CASE1 is to compile ONOS and push it to the test machines |
| 44 | |
| 45 | Startup sequence: |
| 46 | cell <name> |
| 47 | onos-verify-cell |
| 48 | NOTE: temporary - onos-remove-raft-logs |
| 49 | onos-uninstall |
| 50 | start mininet |
| 51 | git pull |
| 52 | mvn clean install |
| 53 | onos-package |
| 54 | onos-install -f |
| 55 | onos-wait-for-start |
| 56 | start cli sessions |
| 57 | start tcpdump |
| 58 | """ |
| 59 | import imp |
| 60 | import time |
| 61 | import json |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 62 | try: |
| 63 | from tests.dependencies.ONOSSetup import ONOSSetup |
| 64 | main.testSetUp = ONOSSetup() |
| 65 | except ImportError: |
| 66 | main.log.error( "ONOSSetup not found. exiting the test" ) |
| 67 | main.cleanAndExit() |
| 68 | main.testSetUp.envSetupDescription() |
| 69 | stepResult = main.FALSE |
| 70 | try: |
| 71 | # load some variables from the params file |
| 72 | cellName = main.params[ 'ENV' ][ 'cellName' ] |
| 73 | main.timeSleep = int( main.params[ 'RETRY' ][ 'sleep' ] ) |
| 74 | main.numAttempts = int( main.params[ 'RETRY' ][ 'attempts' ] ) |
| 75 | main.apps = main.params[ 'ENV' ][ 'cellApps' ] |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 76 | |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 77 | ofPort = main.params[ 'CTRL' ][ 'port' ] |
| 78 | stepResult = main.testSetUp.envSetup() |
| 79 | except Exception as e: |
| 80 | main.testSetUp.envSetupException( e ) |
| 81 | main.testSetUp.evnSetupConclusion( stepResult ) |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 82 | |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 83 | main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, |
| 84 | cellName=cellName ) |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 85 | |
| 86 | main.step( "Starting Mininet" ) |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 87 | # scp topo file to mininet |
| 88 | # TODO: move to params? |
| 89 | topoName = "vpls" |
Jeremy Ronquillo | 31caa48 | 2017-07-18 15:55:31 -0700 | [diff] [blame] | 90 | topoFile = "vpls.py" |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 91 | filePath = main.ONOSbench.home + "/tools/test/topos/" |
| 92 | main.ONOSbench.scp( main.Mininet1, |
| 93 | filePath + topoFile, |
| 94 | main.Mininet1.home, |
| 95 | direction="to" ) |
| 96 | topo = " --custom " + main.Mininet1.home + topoFile + " --topo " + topoName |
| 97 | args = " --switch ovs,protocols=OpenFlow13" |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 98 | for ctrl in main.Cluster.active(): |
| 99 | args += " --controller=remote,ip=" + ctrl.ipAddress |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 100 | mnCmd = "sudo mn" + topo + args |
| 101 | mnResult = main.Mininet1.startNet( mnCmd=mnCmd ) |
| 102 | utilities.assert_equals( expect=main.TRUE, actual=mnResult, |
| 103 | onpass="Mininet Started", |
| 104 | onfail="Error starting Mininet" ) |
| 105 | |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 106 | |
| 107 | main.step( "Activate apps defined in the params file" ) |
| 108 | # get data from the params |
| 109 | apps = main.params.get( 'apps' ) |
| 110 | if apps: |
| 111 | apps = apps.split( ',' ) |
| 112 | main.log.warn( apps ) |
| 113 | activateResult = True |
| 114 | for app in apps: |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 115 | main.Cluster.active( 0 ).CLI.app( app, "Activate" ) |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 116 | # TODO: check this worked |
| 117 | time.sleep( SLEEP ) # wait for apps to activate |
| 118 | for app in apps: |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 119 | state = main.Cluster.active( 0 ).CLI.appStatus( app ) |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 120 | if state == "ACTIVE": |
| 121 | activateResult = activateResult and True |
| 122 | else: |
| 123 | main.log.error( "{} is in {} state".format( app, state ) ) |
| 124 | activateResult = False |
| 125 | utilities.assert_equals( expect=True, |
| 126 | actual=activateResult, |
| 127 | onpass="Successfully activated apps", |
| 128 | onfail="Failed to activate apps" ) |
| 129 | else: |
| 130 | main.log.warn( "No apps were specified to be loaded after startup" ) |
| 131 | |
| 132 | main.step( "Set ONOS configurations" ) |
| 133 | config = main.params.get( 'ONOS_Configuration' ) |
| 134 | if config: |
| 135 | main.log.debug( config ) |
| 136 | checkResult = main.TRUE |
| 137 | for component in config: |
| 138 | for setting in config[ component ]: |
| 139 | value = config[ component ][ setting ] |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 140 | check = main.Cluster.active( 0 ).CLI.setCfg( component, setting, value ) |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 141 | main.log.info( "Value was changed? {}".format( main.TRUE == check ) ) |
| 142 | checkResult = check and checkResult |
| 143 | utilities.assert_equals( expect=main.TRUE, |
| 144 | actual=checkResult, |
| 145 | onpass="Successfully set config", |
| 146 | onfail="Failed to set config" ) |
| 147 | else: |
| 148 | main.log.warn( "No configurations were specified to be changed after startup" ) |
| 149 | |
| 150 | main.step( "App Ids check" ) |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 151 | appCheck = main.Cluster.command( "appToIDCheck", returnBool=True ) |
| 152 | if appCheck != True: |
| 153 | main.log.warn( main.Cluster.active( 0 ).CLI.apps() ) |
| 154 | main.log.warn( main.Cluster.active( 0 ).CLI.appIDs() ) |
| 155 | utilities.assert_equals( expect=True, actual=appCheck, |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 156 | onpass="App Ids seem to be correct", |
| 157 | onfail="Something is wrong with app Ids" ) |
| 158 | |
| 159 | def CASE2( self, main ): |
| 160 | """ |
| 161 | Load and test vpls configurations from json configuration file |
| 162 | """ |
| 163 | import os.path |
| 164 | from tests.USECASE.VPLS.dependencies import vpls |
| 165 | |
| 166 | main.vpls = vpls |
| 167 | pprint = main.ONOSrest1.pprint |
| 168 | hosts = int( main.params[ 'vpls' ][ 'hosts' ] ) |
| 169 | SLEEP = int( main.params[ 'SLEEP' ][ 'netcfg' ] ) |
| 170 | |
| 171 | main.step( "Discover hosts using pings" ) |
| 172 | for i in range( 1, hosts + 1 ): |
| 173 | src = "h" + str( i ) |
| 174 | for j in range( 1, hosts + 1 ): |
| 175 | if j == i: |
| 176 | continue |
| 177 | dst = "h" + str( j ) |
| 178 | pingResult = main.Mininet1.pingHost( SRC=src, TARGET=dst ) |
| 179 | |
| 180 | main.step( "Load VPLS configurations" ) |
| 181 | fileName = main.params[ 'DEPENDENCY' ][ 'topology' ] |
| 182 | app = main.params[ 'vpls' ][ 'name' ] |
| 183 | |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 184 | loadVPLSResult = main.ONOSbench.onosNetCfg( main.Cluster.active( 0 ).ipAddress, "", fileName ) |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 185 | utilities.assert_equals( expect=main.TRUE, |
| 186 | actual=loadVPLSResult, |
| 187 | onpass="Loaded vpls configuration.", |
| 188 | onfail="Failed to load vpls configuration.") |
| 189 | |
| 190 | # Time for netcfg to load data |
| 191 | time.sleep( SLEEP ) |
| 192 | |
| 193 | main.step( "Check VPLS configurations" ) |
| 194 | # 'Master' copy of test configuration |
| 195 | try: |
| 196 | with open( os.path.expanduser( fileName ) ) as dataFile: |
| 197 | originalCfg = json.load( dataFile ) |
| 198 | main.vplsConfig = originalCfg[ 'apps' ].get( app ).get( 'vpls' ).get( 'vplsList' ) |
| 199 | except Exception as e: |
| 200 | main.log.error( "Error loading config file: {}".format( e ) ) |
| 201 | if main.vplsConfig: |
| 202 | result = True |
| 203 | else: |
| 204 | result = False |
| 205 | utilities.assert_equals( expect=True, |
| 206 | actual=result, |
| 207 | onpass="Check vpls configuration succeeded.", |
| 208 | onfail="Check vpls configuration failed." ) |
| 209 | |
| 210 | main.step( "Check interface configurations" ) |
| 211 | result = False |
| 212 | getPorts = utilities.retry( f=main.ONOSrest1.getNetCfg, |
| 213 | retValue=False, |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 214 | kwargs={ "subjectClass" : "ports" }, |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 215 | sleep=SLEEP ) |
| 216 | onosCfg = pprint( getPorts ) |
| 217 | sentCfg = pprint( originalCfg.get( "ports" ) ) |
| 218 | |
| 219 | if onosCfg == sentCfg: |
| 220 | main.log.info( "ONOS interfaces NetCfg matches what was sent" ) |
| 221 | result = True |
| 222 | else: |
| 223 | main.log.error( "ONOS interfaces NetCfg doesn't match what was sent" ) |
| 224 | main.log.debug( "ONOS config: {}".format( onosCfg ) ) |
| 225 | main.log.debug( "Sent config: {}".format( sentCfg ) ) |
| 226 | utilities.assert_equals( expect=True, |
| 227 | actual=result, |
| 228 | onpass="Net Cfg added for interfaces", |
| 229 | onfail="Net Cfg not added for interfaces" ) |
| 230 | |
| 231 | # Run a bunch of checks to verify functionality based on configs |
| 232 | vpls.verify( main ) |
| 233 | |
| 234 | # This is to avoid a race condition in pushing netcfg's. |
| 235 | main.step( "Loading vpls configuration in case any configuration was missed." ) |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 236 | loadVPLSResult = main.ONOSbench.onosNetCfg( main.Cluster.active( 0 ).ipAddress, "", fileName ) |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 237 | utilities.assert_equals( expect=main.TRUE, |
| 238 | actual=loadVPLSResult, |
| 239 | onpass="Loaded vpls configuration.", |
| 240 | onfail="Failed to load vpls configuration." ) |
| 241 | |
| 242 | |
| 243 | |
| 244 | def CASE50( self, main ): |
| 245 | """ |
| 246 | Initial connectivity check |
| 247 | """ |
| 248 | main.case( "Check connectivity before running all other tests." ) |
| 249 | |
| 250 | # Check connectivity before running all other tests |
| 251 | connectCheckResult = main.vpls.testConnectivityVpls( main ) |
| 252 | utilities.assert_equals( expect=main.TRUE, actual=connectCheckResult, |
| 253 | onpass="Connectivity is as expected.", |
| 254 | onfail="Connectivity is NOT as expected." ) |
| 255 | |
| 256 | |
| 257 | def CASE100( self, main ): |
| 258 | """ |
| 259 | Bring down 1 host at a time and test connectivity |
| 260 | """ |
| 261 | assert vpls, "vpls not defined" |
| 262 | |
| 263 | main.case( "Bring down one host at a time and test connectivity." ) |
| 264 | result = main.TRUE |
| 265 | |
| 266 | for i in range( 1, hosts + 1 ): |
| 267 | |
| 268 | stri = str( i ) |
| 269 | |
| 270 | # Bring host down |
| 271 | main.step( "Kill link between s" + stri + " and h" + stri + "." ) |
| 272 | linkDownResult = main.Mininet1.link( END1="s" + stri, END2="h" + stri, OPTION="down" ) |
| 273 | |
| 274 | # Check if link was successfully down'd |
| 275 | utilities.assert_equals( expect=main.TRUE, actual=linkDownResult, |
| 276 | onpass="Link down successful.", |
| 277 | onfail="Failed to bring link down." ) |
| 278 | result = result and linkDownResult |
| 279 | |
| 280 | # Check connectivity |
| 281 | connectivityResult = vpls.testConnectivityVpls( main, blacklist=[ "h" + stri ] ) |
| 282 | result = result and connectivityResult |
| 283 | |
| 284 | # Bring host up |
| 285 | main.step( "Re-adding link between s" + stri + " and h" + stri + "." ) |
| 286 | linkUpResult = main.Mininet1.link( END1="s" + stri, END2="h" + stri, OPTION="up" ) |
| 287 | |
| 288 | # Check if link was successfully re-added |
| 289 | utilities.assert_equals( expect=main.TRUE, actual=linkUpResult, |
| 290 | onpass="Link up successful.", |
| 291 | onfail="Failed to bring link up." ) |
| 292 | result = result and linkUpResult |
| 293 | |
| 294 | # Discover host using ping |
| 295 | main.step( "Discover h" + stri + " using ping." ) |
| 296 | main.Mininet1.pingHost( SRC="h" + stri, TARGET="h" + str( ( i % hosts ) + 1 ) ) |
| 297 | |
| 298 | # Check connectivity |
| 299 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 300 | result = result and connectivityResult |
| 301 | |
| 302 | utilities.assert_equals( expect=main.TRUE, actual=result, |
| 303 | onpass="Connectivity is as expected.", |
| 304 | onfail="Connectivity is NOT as expected.") |
| 305 | |
| 306 | def CASE200( self, main ): |
| 307 | """ |
| 308 | Bring down 1 switch at a time and test connectivity |
| 309 | """ |
| 310 | assert vpls, "vpls not defined" |
| 311 | |
| 312 | main.case( "Bring down one switch at a time and test connectivity." ) |
| 313 | links = main.Mininet1.getLinks( ) # Obtain links here |
| 314 | result = main.TRUE |
| 315 | |
| 316 | for i in range( 5, hosts + 1 ): |
| 317 | |
| 318 | stri = str( i ) |
| 319 | |
| 320 | # Bring switch down |
| 321 | main.step( "Delete s" + stri + ".") |
| 322 | delSwitchResult = main.Mininet1.delSwitch( sw="s" + stri ) |
| 323 | |
| 324 | # Check if switch was deleted |
| 325 | utilities.assert_equals( expect=main.TRUE, actual=delSwitchResult, |
| 326 | onpass="Successfully deleted switch.", |
| 327 | onfail="Failed to delete switch." ) |
| 328 | result = result and delSwitchResult |
| 329 | |
| 330 | # Check connectivity |
| 331 | connectivityResult = vpls.testConnectivityVpls( main, blacklist=[ "h" + stri ] ) |
| 332 | result = result and connectivityResult |
| 333 | |
| 334 | # Bring switch up |
| 335 | main.step( "Add s" + stri + ".") |
| 336 | addSwitchResult = main.Mininet1.addSwitch( sw="s" + stri ) |
| 337 | |
| 338 | # Check if switch was added |
| 339 | utilities.assert_equals( expect=main.TRUE, actual=addSwitchResult, |
| 340 | onpass="Successfully added switch.", |
| 341 | onfail="Failed to add switch.") |
| 342 | result = result and addSwitchResult |
| 343 | |
| 344 | # Reconnect links |
| 345 | main.step( "Reconnecting links on s" + stri + ".") |
| 346 | for j in links: |
| 347 | if ( j[ 'node1' ] == "s" + stri and j[ 'node2' ][ 0 ] == "s" ) or \ |
| 348 | ( j[ 'node2' ] == "s" + stri and j[ 'node1' ][ 0 ] == "s" ): |
| 349 | main.Mininet1.addLink( str( j[ 'node1' ] ), str( j[ 'node2' ] ) ) |
| 350 | |
| 351 | # Discover host using ping |
| 352 | main.Mininet1.pingHost( SRC="h" + stri, TARGET="h" + str( ( i % hosts ) + 1 ) ) |
| 353 | |
| 354 | # Check connectivity |
| 355 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 356 | result = result and connectivityResult |
| 357 | |
| 358 | |
| 359 | utilities.assert_equals( expect=main.TRUE, |
| 360 | actual=result, |
| 361 | onpass="Connectivity is as expected.", |
| 362 | onfail="Connectivity is NOT as expected." ) |
| 363 | |
| 364 | def CASE300( self, main ): |
| 365 | """ |
| 366 | Stop 1 ONOS node at a time and test connectivity |
| 367 | """ |
| 368 | from tests.USECASE.VPLS.dependencies import vpls |
| 369 | from tests.HA.dependencies.HA import HA |
| 370 | assert vpls, "vpls not defined" |
| 371 | |
| 372 | main.HA = HA() |
| 373 | main.case( "Stop one ONOS node at a time and test connectivity." ) |
| 374 | |
| 375 | result = main.TRUE |
| 376 | |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 377 | for i in range( 0, main.Cluster.numCtrls ): |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 378 | |
| 379 | stri = str( i ) |
| 380 | |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 381 | ip_address = main.Cluster.active( i ).ipAddress |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 382 | |
| 383 | # Stop an ONOS node: i |
| 384 | main.step( "Stop ONOS node " + stri + ".") |
| 385 | stopResult = main.ONOSbench.onosStop( ip_address ) |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 386 | main.Cluster.runningNodes[ i ].active = False |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 387 | |
| 388 | utilities.assert_equals( expect=main.TRUE, actual=stopResult, |
| 389 | onpass="ONOS nodes stopped successfully.", |
| 390 | onfail="ONOS nodes NOT successfully stopped." ) |
| 391 | |
| 392 | # Check connectivity |
| 393 | connectivityResult = vpls.testConnectivityVpls( main, isNodeUp=False ) |
| 394 | result = result and connectivityResult |
| 395 | |
| 396 | # Restart ONOS node |
| 397 | main.step( "Restart ONOS node " + stri + " and checking status of restart.") |
| 398 | startResult = main.ONOSbench.onosStart( ip_address ) |
| 399 | |
| 400 | utilities.assert_equals( expect=main.TRUE, actual=startResult, |
| 401 | onpass="ONOS nodes started successfully.", |
| 402 | onfail="ONOS nodes NOT successfully started." ) |
| 403 | result = result and startResult |
| 404 | |
| 405 | # Check if ONOS is up yet |
| 406 | main.log.info( "Checking if ONOS node " + stri + " is up." ) |
| 407 | upResult = main.ONOSbench.isup( ip_address ) |
| 408 | |
| 409 | utilities.assert_equals( expect=main.TRUE, actual=upResult, |
| 410 | onpass="ONOS successfully restarted.", |
| 411 | onfail="ONOS did NOT successfully restart." ) |
| 412 | |
| 413 | # Restart CLI |
| 414 | main.log.info( "Restarting ONOS node " + stri + "'s main.CLI." ) |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 415 | cliResults = main.Cluster.active( 0 ).CLI.startOnosCli( ip_address ) |
| 416 | main.Cluster.runningNodes[ i ].active = True |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 417 | |
| 418 | utilities.assert_equals( expect=main.TRUE, actual=cliResults, |
| 419 | onpass="ONOS CLI successfully restarted.", |
| 420 | onfail="ONOS CLI did NOT successfully restart." ) |
| 421 | |
| 422 | # Run some basic checks to see if ONOS node truly has succesfully restarted: |
| 423 | |
| 424 | # Checking if all nodes appear with status READY using 'nodes' command |
| 425 | main.step( "Checking ONOS nodes." ) |
| 426 | nodeResults = utilities.retry( main.HA.nodesCheck, |
| 427 | False, |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 428 | args=[ main.Cluster.runningNodes ], |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 429 | sleep=main.timeSleep, |
| 430 | attempts=main.numAttempts ) |
| 431 | |
| 432 | utilities.assert_equals( expect=True, actual=nodeResults, |
| 433 | onpass="Nodes check successful.", |
| 434 | onfail="Nodes check NOT successful." ) |
| 435 | |
| 436 | # All apps that are present are active |
| 437 | main.log.info( "Checking if apps are active." ) |
| 438 | compareAppsResult = vpls.compareApps( main ) |
| 439 | utilities.assert_equals( expect=main.TRUE, |
| 440 | actual=compareAppsResult, |
| 441 | onpass="Apps are the same across all nodes.", |
| 442 | onfail="Apps are NOT the same across all nodes." ) |
| 443 | result = result and compareAppsResult |
| 444 | |
| 445 | # Check connectivity |
| 446 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 447 | result = result and connectivityResult |
| 448 | |
| 449 | utilities.assert_equals( expect=main.TRUE, |
| 450 | actual=result, |
| 451 | onpass="Connectivity is as expected.", |
| 452 | onfail="Connectivity is NOT as expected." ) |
| 453 | |
| 454 | |
| 455 | def CASE310( self, main ): |
| 456 | """ |
| 457 | Kill 1 ONOS node at a time and test connectivity |
| 458 | """ |
| 459 | assert vpls, "vpls not defined" |
| 460 | |
| 461 | main.case( "Kill one ONOS node at a time and test connectivity." ) |
| 462 | killSleep = int( main.params[ 'SLEEP' ][ 'killnode' ] ) |
| 463 | result = main.TRUE |
| 464 | |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 465 | for i in range( 0, main.Cluster.numCtrls ): |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 466 | |
| 467 | # Kill an ONOS node |
| 468 | main.step( "Killing ONOS node " + str( i + 1 ) + "." ) |
Jeremy Ronquillo | d7791d8 | 2017-08-14 16:23:08 +0000 | [diff] [blame] | 469 | killresult = main.ONOSbench.onosKill( main.Cluster.active( i ).ipAddress ) |
Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 470 | |
| 471 | # Check if ONOS node has been successfully killed |
| 472 | utilities.assert_equals( expect=main.TRUE, actual=killresult, |
| 473 | onpass="ONOS node killed successfully.", |
| 474 | onfail="ONOS node NOT successfully killed." ) |
| 475 | |
| 476 | main.step( "Waiting for ONOS to restart." ) |
| 477 | main.log.info( "Sleeping for " + str( killSleep ) + " seconds..." ) |
| 478 | time.sleep( killSleep ) |
| 479 | |
| 480 | # Check connectivity |
| 481 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 482 | result = result and connectivityResult |
| 483 | |
| 484 | utilities.assert_equals( expect=main.TRUE, |
| 485 | actual=result, |
| 486 | onpass="Connectivity is as expected.", |
| 487 | onfail="Connectivity is NOT as expected.") |
| 488 | |
| 489 | |
| 490 | def CASE400( self, main ): |
| 491 | """ |
| 492 | Bring down 1 link at a time and test connectivity |
| 493 | """ |
| 494 | assert vpls, "vpls not defined" |
| 495 | |
| 496 | |
| 497 | main.case( "Bring down one link at a time and test connectivity." ) |
| 498 | |
| 499 | result = main.TRUE |
| 500 | |
| 501 | for link in main.Mininet1.getLinks(): |
| 502 | nodes = [ link[ 'node1' ], link[ 'node2' ] ] |
| 503 | |
| 504 | # Bring down a link |
| 505 | main.step( "Bring down link: " + nodes[ 0 ] + " to " + nodes[ 1 ] ) |
| 506 | delLinkResult = main.Mininet1.link( END1=nodes[ 0 ], END2=nodes[ 1 ], OPTION="down" ) |
| 507 | |
| 508 | # Check if the link has successfully been brought down |
| 509 | utilities.assert_equals( expect=main.TRUE, actual=delLinkResult, |
| 510 | onpass="Successfully deleted link.", |
| 511 | onfail="Failed to delete link." ) |
| 512 | result = result and delLinkResult |
| 513 | |
| 514 | # Add removed host to blacklist if necessary |
| 515 | blacklist = [] |
| 516 | for l in nodes: |
| 517 | if l[ 0 ] == 'h': |
| 518 | blacklist.append( l ) |
| 519 | |
| 520 | # Check intent states, then connectivity |
| 521 | connectivityResult = vpls.testConnectivityVpls( main, blacklist ) |
| 522 | result = result and connectivityResult |
| 523 | |
| 524 | # Re-add the link |
| 525 | main.step( "Adding link: " + nodes[ 0 ] + " to " + nodes[ 1 ] + "." ) |
| 526 | addLinkResult = main.Mininet1.link( END1=nodes[ 0 ], END2=nodes[ 1 ], OPTION="up" ) |
| 527 | |
| 528 | # Check if the link has successfully been added |
| 529 | utilities.assert_equals( expect=main.TRUE, actual=addLinkResult, |
| 530 | onpass="Successfully added link.", |
| 531 | onfail="Failed to delete link." ) |
| 532 | result = result and addLinkResult |
| 533 | |
| 534 | main.log.debug( main.timeSleep ) |
| 535 | time.sleep( main.timeSleep ) |
| 536 | |
| 537 | # Check intent states, then connectivity |
| 538 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 539 | result = result and connectivityResult |
| 540 | |
| 541 | utilities.assert_equals( expect=main.TRUE, |
| 542 | actual=result, |
| 543 | onpass="Connectivity is as expected.", |
| 544 | onfail="Connectivity is NOT as expected." ) |