Jeremy Ronquillo | 3008aa3 | 2017-07-07 15:38:57 -0700 | [diff] [blame] | 1 | """ |
| 2 | Testing various connectivity failures for VPLS. |
| 3 | |
| 4 | CASE1: Startup. |
| 5 | CASE2: Load VPLS topology and configurations from demo script. |
| 6 | CASE50: Initial connectivity test. |
| 7 | CASE100: Bring down 1 host at a time and test connectivity. |
| 8 | CASE200: Bring down 1 switch at a time and test connectivity. |
| 9 | CASE300: Stop 1 ONOS node at a time and test connectivity. |
| 10 | CASE310: Kill 1 ONOS node at a time and test connectivity. |
| 11 | CASE400: Bring down 1 link at a time and test connectivity. |
| 12 | """ |
| 13 | |
| 14 | class VPLSfailsafe: |
| 15 | |
| 16 | def __init__( self ): |
| 17 | self.default = '' |
| 18 | |
| 19 | |
| 20 | def CASE1( self, main ): |
| 21 | """ |
| 22 | CASE1 is to compile ONOS and push it to the test machines |
| 23 | |
| 24 | Startup sequence: |
| 25 | cell <name> |
| 26 | onos-verify-cell |
| 27 | NOTE: temporary - onos-remove-raft-logs |
| 28 | onos-uninstall |
| 29 | start mininet |
| 30 | git pull |
| 31 | mvn clean install |
| 32 | onos-package |
| 33 | onos-install -f |
| 34 | onos-wait-for-start |
| 35 | start cli sessions |
| 36 | start tcpdump |
| 37 | """ |
| 38 | import imp |
| 39 | import time |
| 40 | import json |
| 41 | |
| 42 | main.case( "Setting up test environment" ) |
| 43 | main.caseExplanation = "Setup the test environment including " +\ |
| 44 | "installing ONOS, starting Mininet and ONOS" +\ |
| 45 | "cli sessions." |
| 46 | |
| 47 | # load some variables from the params file |
| 48 | cellName = main.params[ 'ENV' ][ 'cellName' ] |
| 49 | |
| 50 | main.numCtrls = int( main.params[ 'num_controllers' ] ) |
| 51 | |
| 52 | ofPort = main.params[ 'CTRL' ][ 'port' ] |
| 53 | |
| 54 | main.timeSleep = int( main.params[ 'RETRY' ][ 'sleep' ] ) |
| 55 | main.numAttempts = int( main.params[ 'RETRY' ][ 'attempts' ] ) |
| 56 | |
| 57 | main.CLIs = [] |
| 58 | main.RESTs = [] |
| 59 | main.nodes = [] |
| 60 | ipList = [] |
| 61 | for i in range( 1, main.numCtrls + 1 ): |
| 62 | try: |
| 63 | main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) ) |
| 64 | main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) ) |
| 65 | main.nodes.append( getattr( main, 'ONOS' + str( i ) ) ) |
| 66 | ipList.append( main.nodes[ -1 ].ip_address ) |
| 67 | except AttributeError: |
| 68 | break |
| 69 | |
| 70 | main.step( "Create cell file" ) |
| 71 | cellAppString = main.params[ 'ENV' ][ 'cellApps' ] |
| 72 | main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName, |
| 73 | main.Mininet1.ip_address, |
| 74 | cellAppString, ipList, main.ONOScli1.karafUser ) |
| 75 | main.step( "Applying cell variable to environment" ) |
| 76 | main.cellResult = main.ONOSbench.setCell( cellName ) |
| 77 | verifyResult = main.ONOSbench.verifyCell() |
| 78 | |
| 79 | main.log.info( "Uninstalling ONOS" ) |
| 80 | for node in main.nodes: |
| 81 | main.ONOSbench.onosUninstall( node.ip_address ) |
| 82 | |
| 83 | # Make sure ONOS is DEAD |
| 84 | main.log.info( "Killing any ONOS processes" ) |
| 85 | killResults = main.TRUE |
| 86 | for node in main.nodes: |
| 87 | killed = main.ONOSbench.onosKill( node.ip_address ) |
| 88 | killResults = killResults and killed |
| 89 | |
| 90 | main.step( "Starting Mininet" ) |
| 91 | |
| 92 | # scp topo file to mininet |
| 93 | # TODO: move to params? |
| 94 | topoName = "vpls" |
| 95 | topoFile = "vplsfailsafe-topo.py" |
| 96 | filePath = main.ONOSbench.home + "/tools/test/topos/" |
| 97 | main.ONOSbench.scp( main.Mininet1, |
| 98 | filePath + topoFile, |
| 99 | main.Mininet1.home, |
| 100 | direction="to" ) |
| 101 | topo = " --custom " + main.Mininet1.home + topoFile + " --topo " + topoName |
| 102 | args = " --switch ovs,protocols=OpenFlow13" |
| 103 | for node in main.nodes: |
| 104 | args += " --controller=remote,ip=" + node.ip_address |
| 105 | mnCmd = "sudo mn" + topo + args |
| 106 | mnResult = main.Mininet1.startNet( mnCmd=mnCmd ) |
| 107 | utilities.assert_equals( expect=main.TRUE, actual=mnResult, |
| 108 | onpass="Mininet Started", |
| 109 | onfail="Error starting Mininet" ) |
| 110 | |
| 111 | main.ONOSbench.getVersion( report=True ) |
| 112 | |
| 113 | main.step( "Creating ONOS package" ) |
| 114 | packageResult = main.ONOSbench.buckBuild() |
| 115 | utilities.assert_equals( expect=main.TRUE, actual=packageResult, |
| 116 | onpass="ONOS package successful", |
| 117 | onfail="ONOS package failed" ) |
| 118 | |
| 119 | main.step( "Installing ONOS package" ) |
| 120 | onosInstallResult = main.TRUE |
| 121 | for node in main.nodes: |
| 122 | tmpResult = main.ONOSbench.onosInstall( options="-f", |
| 123 | node=node.ip_address ) |
| 124 | onosInstallResult = onosInstallResult and tmpResult |
| 125 | utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult, |
| 126 | onpass="ONOS install successful", |
| 127 | onfail="ONOS install failed" ) |
| 128 | |
| 129 | main.step( "Set up ONOS secure SSH" ) |
| 130 | secureSshResult = main.TRUE |
| 131 | for node in main.nodes: |
| 132 | secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=node.ip_address ) |
| 133 | utilities.assert_equals( expect=main.TRUE, actual=secureSshResult, |
| 134 | onpass="Test step PASS", |
| 135 | onfail="Test step FAIL" ) |
| 136 | |
| 137 | main.step( "Checking if ONOS is up yet" ) |
| 138 | for i in range( 2 ): |
| 139 | onosIsupResult = main.TRUE |
| 140 | for node in main.nodes: |
| 141 | started = main.ONOSbench.isup( node.ip_address ) |
| 142 | if not started: |
| 143 | main.log.error( node.name + " hasn't started" ) |
| 144 | onosIsupResult = onosIsupResult and started |
| 145 | if onosIsupResult == main.TRUE: |
| 146 | break |
| 147 | utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult, |
| 148 | onpass="ONOS startup successful", |
| 149 | onfail="ONOS startup failed" ) |
| 150 | |
| 151 | main.step( "Starting ONOS CLI sessions" ) |
| 152 | cliResults = main.TRUE |
| 153 | threads = [] |
| 154 | for i in range( main.numCtrls ): |
| 155 | t = main.Thread( target=main.CLIs[ i ].startOnosCli, |
| 156 | name="startOnosCli-" + str( i ), |
| 157 | args=[ main.nodes[ i ].ip_address ] ) |
| 158 | threads.append( t ) |
| 159 | t.start() |
| 160 | |
| 161 | for t in threads: |
| 162 | t.join() |
| 163 | cliResults = cliResults and t.result |
| 164 | utilities.assert_equals( expect=main.TRUE, actual=cliResults, |
| 165 | onpass="ONOS cli startup successful", |
| 166 | onfail="ONOS cli startup failed" ) |
| 167 | |
| 168 | main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ] |
| 169 | |
| 170 | main.step( "Activate apps defined in the params file" ) |
| 171 | # get data from the params |
| 172 | apps = main.params.get( 'apps' ) |
| 173 | if apps: |
| 174 | apps = apps.split( ',' ) |
| 175 | main.log.warn( apps ) |
| 176 | activateResult = True |
| 177 | for app in apps: |
| 178 | main.CLIs[ 0 ].app( app, "Activate" ) |
| 179 | # TODO: check this worked |
| 180 | time.sleep( SLEEP ) # wait for apps to activate |
| 181 | for app in apps: |
| 182 | state = main.CLIs[ 0 ].appStatus( app ) |
| 183 | if state == "ACTIVE": |
| 184 | activateResult = activateResult and True |
| 185 | else: |
| 186 | main.log.error( "{} is in {} state".format( app, state ) ) |
| 187 | activateResult = False |
| 188 | utilities.assert_equals( expect=True, |
| 189 | actual=activateResult, |
| 190 | onpass="Successfully activated apps", |
| 191 | onfail="Failed to activate apps" ) |
| 192 | else: |
| 193 | main.log.warn( "No apps were specified to be loaded after startup" ) |
| 194 | |
| 195 | main.step( "Set ONOS configurations" ) |
| 196 | config = main.params.get( 'ONOS_Configuration' ) |
| 197 | if config: |
| 198 | main.log.debug( config ) |
| 199 | checkResult = main.TRUE |
| 200 | for component in config: |
| 201 | for setting in config[ component ]: |
| 202 | value = config[ component ][ setting ] |
| 203 | check = main.CLIs[ 0 ].setCfg( component, setting, value ) |
| 204 | main.log.info( "Value was changed? {}".format( main.TRUE == check ) ) |
| 205 | checkResult = check and checkResult |
| 206 | utilities.assert_equals( expect=main.TRUE, |
| 207 | actual=checkResult, |
| 208 | onpass="Successfully set config", |
| 209 | onfail="Failed to set config" ) |
| 210 | else: |
| 211 | main.log.warn( "No configurations were specified to be changed after startup" ) |
| 212 | |
| 213 | main.step( "App Ids check" ) |
| 214 | appCheck = main.TRUE |
| 215 | threads = [] |
| 216 | for i in main.activeNodes: |
| 217 | t = main.Thread( target=main.CLIs[ i ].appToIDCheck, |
| 218 | name="appToIDCheck-" + str( i ), |
| 219 | args=[] ) |
| 220 | threads.append( t ) |
| 221 | t.start() |
| 222 | |
| 223 | for t in threads: |
| 224 | t.join() |
| 225 | appCheck = appCheck and t.result |
| 226 | if appCheck != main.TRUE: |
| 227 | main.log.warn( main.CLIs[ 0 ].apps() ) |
| 228 | main.log.warn( main.CLIs[ 0 ].appIDs() ) |
| 229 | utilities.assert_equals( expect=main.TRUE, actual=appCheck, |
| 230 | onpass="App Ids seem to be correct", |
| 231 | onfail="Something is wrong with app Ids" ) |
| 232 | |
| 233 | def CASE2( self, main ): |
| 234 | """ |
| 235 | Load and test vpls configurations from json configuration file |
| 236 | """ |
| 237 | import os.path |
| 238 | from tests.USECASE.VPLS.dependencies import vpls |
| 239 | |
| 240 | main.vpls = vpls |
| 241 | pprint = main.ONOSrest1.pprint |
| 242 | hosts = int( main.params[ 'vpls' ][ 'hosts' ] ) |
| 243 | SLEEP = int( main.params[ 'SLEEP' ][ 'netcfg' ] ) |
| 244 | |
| 245 | main.step( "Discover hosts using pings" ) |
| 246 | for i in range( 1, hosts + 1 ): |
| 247 | src = "h" + str( i ) |
| 248 | for j in range( 1, hosts + 1 ): |
| 249 | if j == i: |
| 250 | continue |
| 251 | dst = "h" + str( j ) |
| 252 | pingResult = main.Mininet1.pingHost( SRC=src, TARGET=dst ) |
| 253 | |
| 254 | main.step( "Load VPLS configurations" ) |
| 255 | fileName = main.params[ 'DEPENDENCY' ][ 'topology' ] |
| 256 | app = main.params[ 'vpls' ][ 'name' ] |
| 257 | |
| 258 | loadVPLSResult = main.ONOSbench.onosNetCfg( main.nodes[ 0 ].ip_address, "", fileName ) |
| 259 | utilities.assert_equals( expect=main.TRUE, |
| 260 | actual=loadVPLSResult, |
| 261 | onpass="Loaded vpls configuration.", |
| 262 | onfail="Failed to load vpls configuration.") |
| 263 | |
| 264 | # Time for netcfg to load data |
| 265 | time.sleep( SLEEP ) |
| 266 | |
| 267 | main.step( "Check VPLS configurations" ) |
| 268 | # 'Master' copy of test configuration |
| 269 | try: |
| 270 | with open( os.path.expanduser( fileName ) ) as dataFile: |
| 271 | originalCfg = json.load( dataFile ) |
| 272 | main.vplsConfig = originalCfg[ 'apps' ].get( app ).get( 'vpls' ).get( 'vplsList' ) |
| 273 | except Exception as e: |
| 274 | main.log.error( "Error loading config file: {}".format( e ) ) |
| 275 | if main.vplsConfig: |
| 276 | result = True |
| 277 | else: |
| 278 | result = False |
| 279 | utilities.assert_equals( expect=True, |
| 280 | actual=result, |
| 281 | onpass="Check vpls configuration succeeded.", |
| 282 | onfail="Check vpls configuration failed." ) |
| 283 | |
| 284 | main.step( "Check interface configurations" ) |
| 285 | result = False |
| 286 | getPorts = utilities.retry( f=main.ONOSrest1.getNetCfg, |
| 287 | retValue=False, |
| 288 | kwargs={"subjectClass":"ports"}, |
| 289 | sleep=SLEEP ) |
| 290 | onosCfg = pprint( getPorts ) |
| 291 | sentCfg = pprint( originalCfg.get( "ports" ) ) |
| 292 | |
| 293 | if onosCfg == sentCfg: |
| 294 | main.log.info( "ONOS interfaces NetCfg matches what was sent" ) |
| 295 | result = True |
| 296 | else: |
| 297 | main.log.error( "ONOS interfaces NetCfg doesn't match what was sent" ) |
| 298 | main.log.debug( "ONOS config: {}".format( onosCfg ) ) |
| 299 | main.log.debug( "Sent config: {}".format( sentCfg ) ) |
| 300 | utilities.assert_equals( expect=True, |
| 301 | actual=result, |
| 302 | onpass="Net Cfg added for interfaces", |
| 303 | onfail="Net Cfg not added for interfaces" ) |
| 304 | |
| 305 | # Run a bunch of checks to verify functionality based on configs |
| 306 | vpls.verify( main ) |
| 307 | |
| 308 | # This is to avoid a race condition in pushing netcfg's. |
| 309 | main.step( "Loading vpls configuration in case any configuration was missed." ) |
| 310 | loadVPLSResult = main.ONOSbench.onosNetCfg( main.nodes[ 0 ].ip_address, "", fileName ) |
| 311 | utilities.assert_equals( expect=main.TRUE, |
| 312 | actual=loadVPLSResult, |
| 313 | onpass="Loaded vpls configuration.", |
| 314 | onfail="Failed to load vpls configuration." ) |
| 315 | |
| 316 | |
| 317 | |
| 318 | def CASE50( self, main ): |
| 319 | """ |
| 320 | Initial connectivity check |
| 321 | """ |
| 322 | main.case( "Check connectivity before running all other tests." ) |
| 323 | |
| 324 | # Check connectivity before running all other tests |
| 325 | connectCheckResult = main.vpls.testConnectivityVpls( main ) |
| 326 | utilities.assert_equals( expect=main.TRUE, actual=connectCheckResult, |
| 327 | onpass="Connectivity is as expected.", |
| 328 | onfail="Connectivity is NOT as expected." ) |
| 329 | |
| 330 | |
| 331 | def CASE100( self, main ): |
| 332 | """ |
| 333 | Bring down 1 host at a time and test connectivity |
| 334 | """ |
| 335 | assert vpls, "vpls not defined" |
| 336 | |
| 337 | main.case( "Bring down one host at a time and test connectivity." ) |
| 338 | result = main.TRUE |
| 339 | |
| 340 | for i in range( 1, hosts + 1 ): |
| 341 | |
| 342 | stri = str( i ) |
| 343 | |
| 344 | # Bring host down |
| 345 | main.step( "Kill link between s" + stri + " and h" + stri + "." ) |
| 346 | linkDownResult = main.Mininet1.link( END1="s" + stri, END2="h" + stri, OPTION="down" ) |
| 347 | |
| 348 | # Check if link was successfully down'd |
| 349 | utilities.assert_equals( expect=main.TRUE, actual=linkDownResult, |
| 350 | onpass="Link down successful.", |
| 351 | onfail="Failed to bring link down." ) |
| 352 | result = result and linkDownResult |
| 353 | |
| 354 | # Check connectivity |
| 355 | connectivityResult = vpls.testConnectivityVpls( main, blacklist=[ "h" + stri ] ) |
| 356 | result = result and connectivityResult |
| 357 | |
| 358 | # Bring host up |
| 359 | main.step( "Re-adding link between s" + stri + " and h" + stri + "." ) |
| 360 | linkUpResult = main.Mininet1.link( END1="s" + stri, END2="h" + stri, OPTION="up" ) |
| 361 | |
| 362 | # Check if link was successfully re-added |
| 363 | utilities.assert_equals( expect=main.TRUE, actual=linkUpResult, |
| 364 | onpass="Link up successful.", |
| 365 | onfail="Failed to bring link up." ) |
| 366 | result = result and linkUpResult |
| 367 | |
| 368 | # Discover host using ping |
| 369 | main.step( "Discover h" + stri + " using ping." ) |
| 370 | main.Mininet1.pingHost( SRC="h" + stri, TARGET="h" + str( ( i % hosts ) + 1 ) ) |
| 371 | |
| 372 | # Check connectivity |
| 373 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 374 | result = result and connectivityResult |
| 375 | |
| 376 | utilities.assert_equals( expect=main.TRUE, actual=result, |
| 377 | onpass="Connectivity is as expected.", |
| 378 | onfail="Connectivity is NOT as expected.") |
| 379 | |
| 380 | def CASE200( self, main ): |
| 381 | """ |
| 382 | Bring down 1 switch at a time and test connectivity |
| 383 | """ |
| 384 | assert vpls, "vpls not defined" |
| 385 | |
| 386 | main.case( "Bring down one switch at a time and test connectivity." ) |
| 387 | links = main.Mininet1.getLinks( ) # Obtain links here |
| 388 | result = main.TRUE |
| 389 | |
| 390 | for i in range( 5, hosts + 1 ): |
| 391 | |
| 392 | stri = str( i ) |
| 393 | |
| 394 | # Bring switch down |
| 395 | main.step( "Delete s" + stri + ".") |
| 396 | delSwitchResult = main.Mininet1.delSwitch( sw="s" + stri ) |
| 397 | |
| 398 | # Check if switch was deleted |
| 399 | utilities.assert_equals( expect=main.TRUE, actual=delSwitchResult, |
| 400 | onpass="Successfully deleted switch.", |
| 401 | onfail="Failed to delete switch." ) |
| 402 | result = result and delSwitchResult |
| 403 | |
| 404 | # Check connectivity |
| 405 | connectivityResult = vpls.testConnectivityVpls( main, blacklist=[ "h" + stri ] ) |
| 406 | result = result and connectivityResult |
| 407 | |
| 408 | # Bring switch up |
| 409 | main.step( "Add s" + stri + ".") |
| 410 | addSwitchResult = main.Mininet1.addSwitch( sw="s" + stri ) |
| 411 | |
| 412 | # Check if switch was added |
| 413 | utilities.assert_equals( expect=main.TRUE, actual=addSwitchResult, |
| 414 | onpass="Successfully added switch.", |
| 415 | onfail="Failed to add switch.") |
| 416 | result = result and addSwitchResult |
| 417 | |
| 418 | # Reconnect links |
| 419 | main.step( "Reconnecting links on s" + stri + ".") |
| 420 | for j in links: |
| 421 | if ( j[ 'node1' ] == "s" + stri and j[ 'node2' ][ 0 ] == "s" ) or \ |
| 422 | ( j[ 'node2' ] == "s" + stri and j[ 'node1' ][ 0 ] == "s" ): |
| 423 | main.Mininet1.addLink( str( j[ 'node1' ] ), str( j[ 'node2' ] ) ) |
| 424 | |
| 425 | # Discover host using ping |
| 426 | main.Mininet1.pingHost( SRC="h" + stri, TARGET="h" + str( ( i % hosts ) + 1 ) ) |
| 427 | |
| 428 | # Check connectivity |
| 429 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 430 | result = result and connectivityResult |
| 431 | |
| 432 | |
| 433 | utilities.assert_equals( expect=main.TRUE, |
| 434 | actual=result, |
| 435 | onpass="Connectivity is as expected.", |
| 436 | onfail="Connectivity is NOT as expected." ) |
| 437 | |
| 438 | def CASE300( self, main ): |
| 439 | """ |
| 440 | Stop 1 ONOS node at a time and test connectivity |
| 441 | """ |
| 442 | from tests.USECASE.VPLS.dependencies import vpls |
| 443 | from tests.HA.dependencies.HA import HA |
| 444 | assert vpls, "vpls not defined" |
| 445 | |
| 446 | main.HA = HA() |
| 447 | main.case( "Stop one ONOS node at a time and test connectivity." ) |
| 448 | |
| 449 | result = main.TRUE |
| 450 | |
| 451 | for i in range( 0, len( main.nodes ) ): |
| 452 | |
| 453 | stri = str( i ) |
| 454 | |
| 455 | ip_address = main.nodes[ i ].ip_address |
| 456 | |
| 457 | # Stop an ONOS node: i |
| 458 | main.step( "Stop ONOS node " + stri + ".") |
| 459 | stopResult = main.ONOSbench.onosStop( ip_address ) |
| 460 | main.activeNodes.remove( i ) |
| 461 | |
| 462 | utilities.assert_equals( expect=main.TRUE, actual=stopResult, |
| 463 | onpass="ONOS nodes stopped successfully.", |
| 464 | onfail="ONOS nodes NOT successfully stopped." ) |
| 465 | |
| 466 | # Check connectivity |
| 467 | connectivityResult = vpls.testConnectivityVpls( main, isNodeUp=False ) |
| 468 | result = result and connectivityResult |
| 469 | |
| 470 | # Restart ONOS node |
| 471 | main.step( "Restart ONOS node " + stri + " and checking status of restart.") |
| 472 | startResult = main.ONOSbench.onosStart( ip_address ) |
| 473 | |
| 474 | utilities.assert_equals( expect=main.TRUE, actual=startResult, |
| 475 | onpass="ONOS nodes started successfully.", |
| 476 | onfail="ONOS nodes NOT successfully started." ) |
| 477 | result = result and startResult |
| 478 | |
| 479 | # Check if ONOS is up yet |
| 480 | main.log.info( "Checking if ONOS node " + stri + " is up." ) |
| 481 | upResult = main.ONOSbench.isup( ip_address ) |
| 482 | |
| 483 | utilities.assert_equals( expect=main.TRUE, actual=upResult, |
| 484 | onpass="ONOS successfully restarted.", |
| 485 | onfail="ONOS did NOT successfully restart." ) |
| 486 | |
| 487 | # Restart CLI |
| 488 | main.log.info( "Restarting ONOS node " + stri + "'s main.CLI." ) |
| 489 | cliResult = main.CLIs[ i ].startOnosCli( ip_address ) |
| 490 | main.activeNodes.append( i ) |
| 491 | |
| 492 | utilities.assert_equals( expect=main.TRUE, actual=cliResults, |
| 493 | onpass="ONOS CLI successfully restarted.", |
| 494 | onfail="ONOS CLI did NOT successfully restart." ) |
| 495 | |
| 496 | # Run some basic checks to see if ONOS node truly has succesfully restarted: |
| 497 | |
| 498 | # Checking if all nodes appear with status READY using 'nodes' command |
| 499 | main.step( "Checking ONOS nodes." ) |
| 500 | nodeResults = utilities.retry( main.HA.nodesCheck, |
| 501 | False, |
| 502 | args=[ main.activeNodes ], |
| 503 | sleep=main.timeSleep, |
| 504 | attempts=main.numAttempts ) |
| 505 | |
| 506 | utilities.assert_equals( expect=True, actual=nodeResults, |
| 507 | onpass="Nodes check successful.", |
| 508 | onfail="Nodes check NOT successful." ) |
| 509 | |
| 510 | # All apps that are present are active |
| 511 | main.log.info( "Checking if apps are active." ) |
| 512 | compareAppsResult = vpls.compareApps( main ) |
| 513 | utilities.assert_equals( expect=main.TRUE, |
| 514 | actual=compareAppsResult, |
| 515 | onpass="Apps are the same across all nodes.", |
| 516 | onfail="Apps are NOT the same across all nodes." ) |
| 517 | result = result and compareAppsResult |
| 518 | |
| 519 | # Check connectivity |
| 520 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 521 | result = result and connectivityResult |
| 522 | |
| 523 | utilities.assert_equals( expect=main.TRUE, |
| 524 | actual=result, |
| 525 | onpass="Connectivity is as expected.", |
| 526 | onfail="Connectivity is NOT as expected." ) |
| 527 | |
| 528 | |
| 529 | def CASE310( self, main ): |
| 530 | """ |
| 531 | Kill 1 ONOS node at a time and test connectivity |
| 532 | """ |
| 533 | assert vpls, "vpls not defined" |
| 534 | |
| 535 | main.case( "Kill one ONOS node at a time and test connectivity." ) |
| 536 | killSleep = int( main.params[ 'SLEEP' ][ 'killnode' ] ) |
| 537 | result = main.TRUE |
| 538 | |
| 539 | for i in range( 0, len( main.nodes ) ): |
| 540 | |
| 541 | # Kill an ONOS node |
| 542 | main.step( "Killing ONOS node " + str( i + 1 ) + "." ) |
| 543 | killresult = main.ONOSbench.onosKill( main.nodes[ i ].ip_address ) |
| 544 | |
| 545 | # Check if ONOS node has been successfully killed |
| 546 | utilities.assert_equals( expect=main.TRUE, actual=killresult, |
| 547 | onpass="ONOS node killed successfully.", |
| 548 | onfail="ONOS node NOT successfully killed." ) |
| 549 | |
| 550 | main.step( "Waiting for ONOS to restart." ) |
| 551 | main.log.info( "Sleeping for " + str( killSleep ) + " seconds..." ) |
| 552 | time.sleep( killSleep ) |
| 553 | |
| 554 | # Check connectivity |
| 555 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 556 | result = result and connectivityResult |
| 557 | |
| 558 | utilities.assert_equals( expect=main.TRUE, |
| 559 | actual=result, |
| 560 | onpass="Connectivity is as expected.", |
| 561 | onfail="Connectivity is NOT as expected.") |
| 562 | |
| 563 | |
| 564 | def CASE400( self, main ): |
| 565 | """ |
| 566 | Bring down 1 link at a time and test connectivity |
| 567 | """ |
| 568 | assert vpls, "vpls not defined" |
| 569 | |
| 570 | |
| 571 | main.case( "Bring down one link at a time and test connectivity." ) |
| 572 | |
| 573 | result = main.TRUE |
| 574 | |
| 575 | for link in main.Mininet1.getLinks(): |
| 576 | nodes = [ link[ 'node1' ], link[ 'node2' ] ] |
| 577 | |
| 578 | # Bring down a link |
| 579 | main.step( "Bring down link: " + nodes[ 0 ] + " to " + nodes[ 1 ] ) |
| 580 | delLinkResult = main.Mininet1.link( END1=nodes[ 0 ], END2=nodes[ 1 ], OPTION="down" ) |
| 581 | |
| 582 | # Check if the link has successfully been brought down |
| 583 | utilities.assert_equals( expect=main.TRUE, actual=delLinkResult, |
| 584 | onpass="Successfully deleted link.", |
| 585 | onfail="Failed to delete link." ) |
| 586 | result = result and delLinkResult |
| 587 | |
| 588 | # Add removed host to blacklist if necessary |
| 589 | blacklist = [] |
| 590 | for l in nodes: |
| 591 | if l[ 0 ] == 'h': |
| 592 | blacklist.append( l ) |
| 593 | |
| 594 | # Check intent states, then connectivity |
| 595 | connectivityResult = vpls.testConnectivityVpls( main, blacklist ) |
| 596 | result = result and connectivityResult |
| 597 | |
| 598 | # Re-add the link |
| 599 | main.step( "Adding link: " + nodes[ 0 ] + " to " + nodes[ 1 ] + "." ) |
| 600 | addLinkResult = main.Mininet1.link( END1=nodes[ 0 ], END2=nodes[ 1 ], OPTION="up" ) |
| 601 | |
| 602 | # Check if the link has successfully been added |
| 603 | utilities.assert_equals( expect=main.TRUE, actual=addLinkResult, |
| 604 | onpass="Successfully added link.", |
| 605 | onfail="Failed to delete link." ) |
| 606 | result = result and addLinkResult |
| 607 | |
| 608 | main.log.debug( main.timeSleep ) |
| 609 | time.sleep( main.timeSleep ) |
| 610 | |
| 611 | # Check intent states, then connectivity |
| 612 | connectivityResult = vpls.testConnectivityVpls( main ) |
| 613 | result = result and connectivityResult |
| 614 | |
| 615 | utilities.assert_equals( expect=main.TRUE, |
| 616 | actual=result, |
| 617 | onpass="Connectivity is as expected.", |
| 618 | onfail="Connectivity is NOT as expected." ) |