[ONOS-6747] New TestON test: VPLSfailsafe and various VPLS test fixes

Change-Id: If88512777256880ea3e627502473ce4eb336292a
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index aa2b563..6923a29 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -2490,12 +2490,11 @@
             cmd = "onos-netcfg {0} {1}{2}".format( controllerIp, path, fileName )
             main.log.info( "Sending: " + cmd )
             main.ONOSbench.handle.sendline( cmd )
+            main.ONOSbench.handle.expect( self.prompt )
             handle = self.handle.before
-            if "Error" in handle:
-                main.log.error( self.name + ":    " + self.handle.before )
+            if "Error" in handle or "No such file or directory" in handle or "curl: " in handle:
+                main.log.error( self.name + ":    " + handle + self.handle.after )
                 return main.FALSE
-            else:
-                self.handle.expect( self.prompt )
             return main.TRUE
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
diff --git a/TestON/tests/USECASE/VPLS/VPLSBasic/VPLSBasic.py b/TestON/tests/USECASE/VPLS/VPLSBasic/VPLSBasic.py
index 62889c9..4b4c1ef 100644
--- a/TestON/tests/USECASE/VPLS/VPLSBasic/VPLSBasic.py
+++ b/TestON/tests/USECASE/VPLS/VPLSBasic/VPLSBasic.py
@@ -85,9 +85,9 @@
                             main.Mininet1.home,
                             direction="to" )
         topo = " --custom " + main.Mininet1.home + topoFile + " --topo " + topoName
-        args = " --switch ovs,protocols=OpenFlow13 --controller=remote"
+        args = " --switch ovs,protocols=OpenFlow13"
         for node in main.nodes:
-            args += ",ip=" + node.ip_address
+            args += " --controller=remote,ip=" + node.ip_address
         mnCmd = "sudo mn" + topo + args
         mnResult = main.Mininet1.startNet( mnCmd=mnCmd )
         utilities.assert_equals( expect=main.TRUE, actual=mnResult,
@@ -223,6 +223,7 @@
         import os.path
         from tests.USECASE.VPLS.dependencies import vpls
 
+        main.vpls = vpls
         pprint = main.ONOSrest1.pprint
         hosts = int( main.params[ 'vpls' ][ 'hosts' ] )
         SLEEP = int( main.params[ 'SLEEP' ][ 'netcfg' ] )
@@ -237,14 +238,19 @@
             pingResult = main.Mininet1.pingHost( SRC=src, TARGET=dst )
 
         main.step( "Load VPLS configurations" )
-        # TODO: load from params
         fileName = main.params[ 'DEPENDENCY' ][ 'topology' ]
         app = main.params[ 'vpls' ][ 'name' ]
-        # TODO make this a function?
-        main.ONOSbench.handle.sendline( "onos-netcfg $OC1 " + fileName )
+
+        loadVPLSResult = main.ONOSbench.onosNetCfg( main.nodes[ 0 ].ip_address, "", fileName )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=loadVPLSResult,
+                                 onpass="Loaded vpls configuration.",
+                                 onfail="Failed to load vpls configuration." )
+
         # Time for netcfg to load data
         time.sleep( SLEEP )
         # 'Master' copy of test configuration
+
         try:
             with open( os.path.expanduser( fileName ) ) as dataFile:
                 originalCfg = json.load( dataFile )
@@ -284,6 +290,13 @@
         # Run a bunch of checks to verify functionality based on configs
         vpls.verify( main )
 
+        main.step( "Loading vpls configuration in case any configuration was missed." )
+        loadVPLSResult = main.ONOSbench.onosNetCfg( main.nodes[ 0 ].ip_address, "", fileName )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=loadVPLSResult,
+                                 onpass="Loaded vpls configuration.",
+                                 onfail="Failed to load vpls configuration." )
+
     def CASE3( self, main ):
         """
         Test VPLS cli commands
diff --git a/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.params b/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.params
new file mode 100755
index 0000000..53e4c6f
--- /dev/null
+++ b/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.params
@@ -0,0 +1,47 @@
+<PARAMS>
+
+    <testcases>1,2,50,100,200,300,310,400</testcases>
+
+    <num_controllers>3</num_controllers>
+
+    <GIT>
+        <pull>False</pull>
+        <branch>master</branch>
+    </GIT>
+
+    <path>/tests/USECASE/VPLS/VPLSfailsafe/dependencies/</path>
+
+    <wrapper2>vpls</wrapper2>
+
+    <DEPENDENCY>
+        <wrapper1>startUp</wrapper1>
+        <topology>~/onos/tools/test/topos/vpls.json</topology>
+    </DEPENDENCY>
+
+    <ENV>
+        <cellName>vpls</cellName>
+        <cellApps>drivers,openflow,vpls</cellApps>
+        <cellUser>sdn</cellUser>
+    </ENV>
+
+    <CTRL>
+        <port>6653</port>
+    </CTRL>
+
+    <vpls>
+        <name>org.onosproject.vpls</name>
+        <hosts>6</hosts>
+    </vpls>
+
+    <SLEEP>
+        <startup>10</startup>
+        <netcfg>10</netcfg>
+        <killnode>10</killnode>
+    </SLEEP>
+
+    <RETRY>
+        <sleep>2</sleep>
+        <attempts>6</attempts>
+    </RETRY>
+
+</PARAMS>
diff --git a/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.py b/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.py
new file mode 100644
index 0000000..f827769
--- /dev/null
+++ b/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.py
@@ -0,0 +1,618 @@
+"""
+Testing various connectivity failures for VPLS.
+
+CASE1: Startup.
+CASE2: Load VPLS topology and configurations from demo script.
+CASE50: Initial connectivity test.
+CASE100: Bring down 1 host at a time and test connectivity.
+CASE200: Bring down 1 switch at a time and test connectivity.
+CASE300: Stop 1 ONOS node at a time and test connectivity.
+CASE310: Kill 1 ONOS node at a time and test connectivity.
+CASE400: Bring down 1 link at a time and test connectivity.
+"""
+
+class VPLSfailsafe:
+
+    def __init__( self ):
+        self.default = ''
+
+
+    def CASE1( self, main ):
+        """
+        CASE1 is to compile ONOS and push it to the test machines
+
+        Startup sequence:
+        cell <name>
+        onos-verify-cell
+        NOTE: temporary - onos-remove-raft-logs
+        onos-uninstall
+        start mininet
+        git pull
+        mvn clean install
+        onos-package
+        onos-install -f
+        onos-wait-for-start
+        start cli sessions
+        start tcpdump
+        """
+        import imp
+        import time
+        import json
+
+        main.case( "Setting up test environment" )
+        main.caseExplanation = "Setup the test environment including " +\
+                                "installing ONOS, starting Mininet and ONOS" +\
+                                "cli sessions."
+
+        # load some variables from the params file
+        cellName = main.params[ 'ENV' ][ 'cellName' ]
+
+        main.numCtrls = int( main.params[ 'num_controllers' ] )
+
+        ofPort = main.params[ 'CTRL' ][ 'port' ]
+
+        main.timeSleep = int( main.params[ 'RETRY' ][ 'sleep' ] )
+        main.numAttempts = int( main.params[ 'RETRY' ][ 'attempts' ] )
+
+        main.CLIs = []
+        main.RESTs = []
+        main.nodes = []
+        ipList = []
+        for i in range( 1, main.numCtrls + 1 ):
+            try:
+                main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+                main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) )
+                main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
+                ipList.append( main.nodes[ -1 ].ip_address )
+            except AttributeError:
+                break
+
+        main.step( "Create cell file" )
+        cellAppString = main.params[ 'ENV' ][ 'cellApps' ]
+        main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
+                                       main.Mininet1.ip_address,
+                                       cellAppString, ipList, main.ONOScli1.karafUser )
+        main.step( "Applying cell variable to environment" )
+        main.cellResult = main.ONOSbench.setCell( cellName )
+        verifyResult = main.ONOSbench.verifyCell()
+
+        main.log.info( "Uninstalling ONOS" )
+        for node in main.nodes:
+            main.ONOSbench.onosUninstall( node.ip_address )
+
+        # Make sure ONOS is DEAD
+        main.log.info( "Killing any ONOS processes" )
+        killResults = main.TRUE
+        for node in main.nodes:
+            killed = main.ONOSbench.onosKill( node.ip_address )
+            killResults = killResults and killed
+
+        main.step( "Starting Mininet" )
+
+        # scp topo file to mininet
+        # TODO: move to params?
+        topoName = "vpls"
+        topoFile = "vplsfailsafe-topo.py"
+        filePath = main.ONOSbench.home + "/tools/test/topos/"
+        main.ONOSbench.scp( main.Mininet1,
+                            filePath + topoFile,
+                            main.Mininet1.home,
+                            direction="to" )
+        topo = " --custom " + main.Mininet1.home + topoFile + " --topo " + topoName
+        args = " --switch ovs,protocols=OpenFlow13"
+        for node in main.nodes:
+            args += " --controller=remote,ip=" + node.ip_address
+        mnCmd = "sudo mn" + topo + args
+        mnResult = main.Mininet1.startNet( mnCmd=mnCmd )
+        utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+                                 onpass="Mininet Started",
+                                 onfail="Error starting Mininet" )
+
+        main.ONOSbench.getVersion( report=True )
+
+        main.step( "Creating ONOS package" )
+        packageResult = main.ONOSbench.buckBuild()
+        utilities.assert_equals( expect=main.TRUE, actual=packageResult,
+                                 onpass="ONOS package successful",
+                                 onfail="ONOS package failed" )
+
+        main.step( "Installing ONOS package" )
+        onosInstallResult = main.TRUE
+        for node in main.nodes:
+            tmpResult = main.ONOSbench.onosInstall( options="-f",
+                                                    node=node.ip_address )
+            onosInstallResult = onosInstallResult and tmpResult
+        utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
+                                 onpass="ONOS install successful",
+                                 onfail="ONOS install failed" )
+
+        main.step( "Set up ONOS secure SSH" )
+        secureSshResult = main.TRUE
+        for node in main.nodes:
+            secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=node.ip_address )
+        utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
+                                 onpass="Test step PASS",
+                                 onfail="Test step FAIL" )
+
+        main.step( "Checking if ONOS is up yet" )
+        for i in range( 2 ):
+            onosIsupResult = main.TRUE
+            for node in main.nodes:
+                started = main.ONOSbench.isup( node.ip_address )
+                if not started:
+                    main.log.error( node.name + " hasn't started" )
+                onosIsupResult = onosIsupResult and started
+            if onosIsupResult == main.TRUE:
+                break
+        utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+                                 onpass="ONOS startup successful",
+                                 onfail="ONOS startup failed" )
+
+        main.step( "Starting ONOS CLI sessions" )
+        cliResults = main.TRUE
+        threads = []
+        for i in range( main.numCtrls ):
+            t = main.Thread( target=main.CLIs[ i ].startOnosCli,
+                             name="startOnosCli-" + str( i ),
+                             args=[ main.nodes[ i ].ip_address ] )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            cliResults = cliResults and t.result
+        utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+                                 onpass="ONOS cli startup successful",
+                                 onfail="ONOS cli startup failed" )
+
+        main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
+
+        main.step( "Activate apps defined in the params file" )
+        # get data from the params
+        apps = main.params.get( 'apps' )
+        if apps:
+            apps = apps.split( ',' )
+            main.log.warn( apps )
+            activateResult = True
+            for app in apps:
+                main.CLIs[ 0 ].app( app, "Activate" )
+            # TODO: check this worked
+            time.sleep( SLEEP )  # wait for apps to activate
+            for app in apps:
+                state = main.CLIs[ 0 ].appStatus( app )
+                if state == "ACTIVE":
+                    activateResult = activateResult and True
+                else:
+                    main.log.error( "{} is in {} state".format( app, state ) )
+                    activateResult = False
+            utilities.assert_equals( expect=True,
+                                     actual=activateResult,
+                                     onpass="Successfully activated apps",
+                                     onfail="Failed to activate apps" )
+        else:
+            main.log.warn( "No apps were specified to be loaded after startup" )
+
+        main.step( "Set ONOS configurations" )
+        config = main.params.get( 'ONOS_Configuration' )
+        if config:
+            main.log.debug( config )
+            checkResult = main.TRUE
+            for component in config:
+                for setting in config[ component ]:
+                    value = config[ component ][ setting ]
+                    check = main.CLIs[ 0 ].setCfg( component, setting, value )
+                    main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
+                    checkResult = check and checkResult
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=checkResult,
+                                     onpass="Successfully set config",
+                                     onfail="Failed to set config" )
+        else:
+            main.log.warn( "No configurations were specified to be changed after startup" )
+
+        main.step( "App Ids check" )
+        appCheck = main.TRUE
+        threads = []
+        for i in main.activeNodes:
+            t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
+                             name="appToIDCheck-" + str( i ),
+                             args=[] )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            appCheck = appCheck and t.result
+        if appCheck != main.TRUE:
+            main.log.warn( main.CLIs[ 0 ].apps() )
+            main.log.warn( main.CLIs[ 0 ].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
+
+    def CASE2( self, main ):
+        """
+        Load and test vpls configurations from json configuration file
+        """
+        import os.path
+        from tests.USECASE.VPLS.dependencies import vpls
+
+        main.vpls = vpls
+        pprint = main.ONOSrest1.pprint
+        hosts = int( main.params[ 'vpls' ][ 'hosts' ] )
+        SLEEP = int( main.params[ 'SLEEP' ][ 'netcfg' ] )
+
+        main.step( "Discover hosts using pings" )
+        for i in range( 1, hosts + 1 ):
+            src = "h" + str( i )
+            for j in range( 1, hosts + 1 ):
+                if j == i:
+                    continue
+                dst = "h" + str( j )
+            pingResult = main.Mininet1.pingHost( SRC=src, TARGET=dst )
+
+        main.step( "Load VPLS configurations" )
+        fileName = main.params[ 'DEPENDENCY' ][ 'topology' ]
+        app = main.params[ 'vpls' ][ 'name' ]
+
+        loadVPLSResult = main.ONOSbench.onosNetCfg( main.nodes[ 0 ].ip_address, "", fileName )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=loadVPLSResult,
+                                 onpass="Loaded vpls configuration.",
+                                 onfail="Failed to load vpls configuration.")
+
+        # Time for netcfg to load data
+        time.sleep( SLEEP )
+
+        main.step( "Check VPLS configurations" )
+        # 'Master' copy of test configuration
+        try:
+            with open( os.path.expanduser( fileName ) ) as dataFile:
+                originalCfg = json.load( dataFile )
+                main.vplsConfig = originalCfg[ 'apps' ].get( app ).get( 'vpls' ).get( 'vplsList' )
+        except Exception as e:
+            main.log.error( "Error loading config file: {}".format( e ) )
+        if main.vplsConfig:
+            result = True
+        else:
+            result = False
+        utilities.assert_equals( expect=True,
+                                 actual=result,
+                                 onpass="Check vpls configuration succeeded.",
+                                 onfail="Check vpls configuration failed." )
+
+        main.step( "Check interface configurations" )
+        result = False
+        getPorts = utilities.retry( f=main.ONOSrest1.getNetCfg,
+                                    retValue=False,
+                                    kwargs={"subjectClass":"ports"},
+                                    sleep=SLEEP )
+        onosCfg = pprint( getPorts )
+        sentCfg = pprint( originalCfg.get( "ports" ) )
+
+        if onosCfg == sentCfg:
+            main.log.info( "ONOS interfaces NetCfg matches what was sent" )
+            result = True
+        else:
+            main.log.error( "ONOS interfaces NetCfg doesn't match what was sent" )
+            main.log.debug( "ONOS config: {}".format( onosCfg ) )
+            main.log.debug( "Sent config: {}".format( sentCfg ) )
+        utilities.assert_equals( expect=True,
+                                 actual=result,
+                                 onpass="Net Cfg added for interfaces",
+                                 onfail="Net Cfg not added for interfaces" )
+
+        # Run a bunch of checks to verify functionality based on configs
+        vpls.verify( main )
+
+        # This is to avoid a race condition in pushing netcfg's.
+        main.step( "Loading vpls configuration in case any configuration was missed." )
+        loadVPLSResult = main.ONOSbench.onosNetCfg( main.nodes[ 0 ].ip_address, "", fileName )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=loadVPLSResult,
+                                 onpass="Loaded vpls configuration.",
+                                 onfail="Failed to load vpls configuration." )
+
+
+
+    def CASE50( self, main ):
+        """
+        Initial connectivity check
+        """
+        main.case( "Check connectivity before running all other tests." )
+
+        # Check connectivity before running all other tests
+        connectCheckResult = main.vpls.testConnectivityVpls( main )
+        utilities.assert_equals( expect=main.TRUE, actual=connectCheckResult,
+                                 onpass="Connectivity is as expected.",
+                                 onfail="Connectivity is NOT as expected." )
+
+
+    def CASE100( self, main ):
+        """
+        Bring down 1 host at a time and test connectivity
+        """
+        assert vpls, "vpls not defined"
+
+        main.case( "Bring down one host at a time and test connectivity." )
+        result = main.TRUE
+
+        for i in range( 1, hosts + 1 ):
+
+            stri = str( i )
+
+            # Bring host down
+            main.step( "Kill link between s" + stri + " and h" + stri + "." )
+            linkDownResult = main.Mininet1.link( END1="s" + stri, END2="h" + stri, OPTION="down" )
+
+            # Check if link was successfully down'd
+            utilities.assert_equals( expect=main.TRUE, actual=linkDownResult,
+                                     onpass="Link down successful.",
+                                     onfail="Failed to bring link down." )
+            result = result and linkDownResult
+
+            # Check connectivity
+            connectivityResult = vpls.testConnectivityVpls( main, blacklist=[ "h" + stri ] )
+            result = result and connectivityResult
+
+            # Bring host up
+            main.step( "Re-adding link between s" + stri + " and h" + stri + "." )
+            linkUpResult = main.Mininet1.link( END1="s" + stri, END2="h" + stri, OPTION="up" )
+
+            # Check if link was successfully re-added
+            utilities.assert_equals( expect=main.TRUE, actual=linkUpResult,
+                                     onpass="Link up successful.",
+                                     onfail="Failed to bring link up." )
+            result = result and linkUpResult
+
+            # Discover host using ping
+            main.step( "Discover h" + stri + " using ping." )
+            main.Mininet1.pingHost( SRC="h" + stri, TARGET="h" + str( ( i % hosts ) + 1 ) )
+
+            # Check connectivity
+            connectivityResult = vpls.testConnectivityVpls( main )
+            result = result and connectivityResult
+
+        utilities.assert_equals( expect=main.TRUE, actual=result,
+                                 onpass="Connectivity is as expected.",
+                                 onfail="Connectivity is NOT as expected.")
+
+    def CASE200( self, main ):
+        """
+        Bring down 1 switch at a time and test connectivity
+        """
+        assert vpls, "vpls not defined"
+
+        main.case( "Bring down one switch at a time and test connectivity." )
+        links = main.Mininet1.getLinks( )  # Obtain links here
+        result = main.TRUE
+
+        for i in range( 5, hosts + 1 ):
+
+            stri = str( i )
+
+            # Bring switch down
+            main.step( "Delete s" + stri + ".")
+            delSwitchResult = main.Mininet1.delSwitch( sw="s" + stri )
+
+            # Check if switch was deleted
+            utilities.assert_equals( expect=main.TRUE, actual=delSwitchResult,
+                                     onpass="Successfully deleted switch.",
+                                     onfail="Failed to delete switch." )
+            result = result and delSwitchResult
+
+            # Check connectivity
+            connectivityResult = vpls.testConnectivityVpls( main, blacklist=[ "h" + stri ] )
+            result = result and connectivityResult
+
+            # Bring switch up
+            main.step( "Add s" + stri + ".")
+            addSwitchResult = main.Mininet1.addSwitch( sw="s" + stri )
+
+            # Check if switch was added
+            utilities.assert_equals( expect=main.TRUE, actual=addSwitchResult,
+                                     onpass="Successfully added switch.",
+                                     onfail="Failed to add switch.")
+            result = result and addSwitchResult
+
+            # Reconnect links
+            main.step( "Reconnecting links on s" + stri + ".")
+            for j in links:
+                if ( j[ 'node1' ] == "s" + stri and j[ 'node2' ][ 0 ] == "s" ) or \
+                   ( j[ 'node2' ] == "s" + stri and j[ 'node1' ][ 0 ] == "s" ):
+                    main.Mininet1.addLink( str( j[ 'node1' ] ), str( j[ 'node2' ] ) )
+
+            # Discover host using ping
+            main.Mininet1.pingHost( SRC="h" + stri, TARGET="h" + str( ( i % hosts ) + 1 ) )
+
+            # Check connectivity
+            connectivityResult = vpls.testConnectivityVpls( main )
+            result = result and connectivityResult
+
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=result,
+                                 onpass="Connectivity is as expected.",
+                                 onfail="Connectivity is NOT as expected." )
+
+    def CASE300( self, main ):
+        """
+        Stop 1 ONOS node at a time and test connectivity
+        """
+        from tests.USECASE.VPLS.dependencies import vpls
+        from tests.HA.dependencies.HA import HA
+        assert vpls, "vpls not defined"
+
+        main.HA = HA()
+        main.case( "Stop one ONOS node at a time and test connectivity." )
+
+        result = main.TRUE
+
+        for i in range( 0, len( main.nodes ) ):
+
+            stri = str( i )
+
+            ip_address = main.nodes[ i ].ip_address
+
+            # Stop an ONOS node: i
+            main.step( "Stop ONOS node " + stri + ".")
+            stopResult = main.ONOSbench.onosStop( ip_address )
+            main.activeNodes.remove( i )
+
+            utilities.assert_equals( expect=main.TRUE, actual=stopResult,
+                                     onpass="ONOS nodes stopped successfully.",
+                                     onfail="ONOS nodes NOT successfully stopped." )
+
+            # Check connectivity
+            connectivityResult = vpls.testConnectivityVpls( main, isNodeUp=False )
+            result = result and connectivityResult
+
+            # Restart ONOS node
+            main.step( "Restart ONOS node " + stri + " and checking status of restart.")
+            startResult = main.ONOSbench.onosStart( ip_address )
+
+            utilities.assert_equals( expect=main.TRUE, actual=startResult,
+                                     onpass="ONOS nodes started successfully.",
+                                     onfail="ONOS nodes NOT successfully started." )
+            result = result and startResult
+
+            # Check if ONOS is up yet
+            main.log.info( "Checking if ONOS node " + stri + " is up." )
+            upResult = main.ONOSbench.isup( ip_address )
+
+            utilities.assert_equals( expect=main.TRUE, actual=upResult,
+                                     onpass="ONOS successfully restarted.",
+                                     onfail="ONOS did NOT successfully restart." )
+
+            # Restart CLI
+            main.log.info( "Restarting ONOS node " + stri + "'s main.CLI." )
+            cliResult = main.CLIs[ i ].startOnosCli( ip_address )
+            main.activeNodes.append( i )
+
+            utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+                                     onpass="ONOS CLI successfully restarted.",
+                                     onfail="ONOS CLI did NOT successfully restart." )
+
+            # Run some basic checks to see if ONOS node truly has succesfully restarted:
+
+            # Checking if all nodes appear with status READY using 'nodes' command
+            main.step( "Checking ONOS nodes." )
+            nodeResults = utilities.retry( main.HA.nodesCheck,
+                                           False,
+                                           args=[ main.activeNodes ],
+                                           sleep=main.timeSleep,
+                                           attempts=main.numAttempts )
+
+            utilities.assert_equals( expect=True, actual=nodeResults,
+                                     onpass="Nodes check successful.",
+                                     onfail="Nodes check NOT successful." )
+
+            # All apps that are present are active
+            main.log.info( "Checking if apps are active." )
+            compareAppsResult = vpls.compareApps( main )
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=compareAppsResult,
+                                     onpass="Apps are the same across all nodes.",
+                                     onfail="Apps are NOT the same across all nodes." )
+            result = result and compareAppsResult
+
+            # Check connectivity
+            connectivityResult = vpls.testConnectivityVpls( main )
+            result = result and connectivityResult
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=result,
+                                 onpass="Connectivity is as expected.",
+                                 onfail="Connectivity is NOT as expected." )
+
+
+    def CASE310( self, main ):
+        """
+        Kill 1 ONOS node at a time and test connectivity
+        """
+        assert vpls, "vpls not defined"
+
+        main.case( "Kill one ONOS node at a time and test connectivity." )
+        killSleep = int( main.params[ 'SLEEP' ][ 'killnode' ] )
+        result = main.TRUE
+
+        for i in range( 0, len( main.nodes ) ):
+
+            # Kill an ONOS node
+            main.step( "Killing ONOS node " + str( i + 1 ) + "." )
+            killresult = main.ONOSbench.onosKill( main.nodes[ i ].ip_address )
+
+            # Check if ONOS node has been successfully killed
+            utilities.assert_equals( expect=main.TRUE, actual=killresult,
+                                     onpass="ONOS node killed successfully.",
+                                     onfail="ONOS node NOT successfully killed." )
+
+            main.step( "Waiting for ONOS to restart." )
+            main.log.info( "Sleeping for " + str( killSleep ) + " seconds..." )
+            time.sleep( killSleep )
+
+            # Check connectivity
+            connectivityResult = vpls.testConnectivityVpls( main )
+            result = result and connectivityResult
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=result,
+                                 onpass="Connectivity is as expected.",
+                                 onfail="Connectivity is NOT as expected.")
+
+
+    def CASE400( self, main ):
+        """
+        Bring down 1 link at a time and test connectivity
+        """
+        assert vpls, "vpls not defined"
+
+
+        main.case( "Bring down one link at a time and test connectivity." )
+
+        result = main.TRUE
+
+        for link in main.Mininet1.getLinks():
+            nodes = [ link[ 'node1' ], link[ 'node2' ] ]
+
+            # Bring down a link
+            main.step( "Bring down link: " + nodes[ 0 ] + " to " + nodes[ 1 ] )
+            delLinkResult = main.Mininet1.link( END1=nodes[ 0 ], END2=nodes[ 1 ], OPTION="down" )
+
+            # Check if the link has successfully been brought down
+            utilities.assert_equals( expect=main.TRUE, actual=delLinkResult,
+                                     onpass="Successfully deleted link.",
+                                     onfail="Failed to delete link." )
+            result = result and delLinkResult
+
+            # Add removed host to blacklist if necessary
+            blacklist = []
+            for l in nodes:
+                if l[ 0 ] == 'h':
+                    blacklist.append( l )
+
+            # Check intent states, then connectivity
+            connectivityResult = vpls.testConnectivityVpls( main, blacklist )
+            result = result and connectivityResult
+
+            # Re-add the link
+            main.step( "Adding link: " + nodes[ 0 ] + " to " + nodes[ 1 ] + "." )
+            addLinkResult = main.Mininet1.link( END1=nodes[ 0 ], END2=nodes[ 1 ], OPTION="up" )
+
+            # Check if the link has successfully been added
+            utilities.assert_equals( expect=main.TRUE, actual=addLinkResult,
+                                     onpass="Successfully added link.",
+                                     onfail="Failed to delete link." )
+            result = result and addLinkResult
+
+            main.log.debug( main.timeSleep )
+            time.sleep( main.timeSleep )
+
+            # Check intent states, then connectivity
+            connectivityResult = vpls.testConnectivityVpls( main )
+            result = result and connectivityResult
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=result,
+                                 onpass="Connectivity is as expected.",
+                                 onfail="Connectivity is NOT as expected." )
diff --git a/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.topo b/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.topo
new file mode 100755
index 0000000..476d986
--- /dev/null
+++ b/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.topo
@@ -0,0 +1,134 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOSbench>
+            <host>localhost</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS>
+                <nodes>1</nodes>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOSbench>
+
+        <ONOScli1>
+            <host>localhost</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS>
+                <karaf_username></karaf_username>
+                <karaf_password></karaf_password>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOScli1>
+
+        <ONOScli2>
+            <host>localhost</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOScli2>
+
+        <ONOScli3>
+            <host>localhost</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOScli3>
+
+        <ONOS1>
+            <host>OC1</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>9</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOS1>
+
+        <ONOS2>
+            <host>OC2</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>10</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOS2>
+
+        <ONOS3>
+            <host>OC3</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>11</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOS3>
+
+        <ONOSrest1>
+            <host>OC1</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOSrest1>
+
+        <ONOSrest2>
+            <host>OC2</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOSrest2>
+
+        <ONOSrest3>
+            <host>OC3</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </ONOSrest3>
+
+
+        <Mininet1>
+            <host>OCN</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>MininetCliDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS>
+                <home>~/mininet/custom/</home>
+                <prompt></prompt>
+            </COMPONENTS>
+        </Mininet1>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/VPLS/dependencies/vpls.py b/TestON/tests/USECASE/VPLS/dependencies/vpls.py
index 02abba2..46e9edf 100644
--- a/TestON/tests/USECASE/VPLS/dependencies/vpls.py
+++ b/TestON/tests/USECASE/VPLS/dependencies/vpls.py
@@ -44,6 +44,8 @@
     NOTE: This requires the expected/sent network config json for the vpls
           application be stored in main.vplsConfig
     """
+
+
     # Variables
     app = main.params[ 'vpls' ][ 'name' ]
     pprint = main.ONOSrest1.pprint
@@ -94,31 +96,8 @@
                              onpass="VPLS successfully configured",
                              onfail="VPLS not configured correctly" )
 
-    # FIXME This doesn't work, some will be withdrawn if interfaces are removed
-    # TODO: if encapsulation is set, look for that
-    # TODO: can we look at the intent keys?
-    """
-    main.step( "Check intent states" )
-    # Print the intent states
-    intents = main.CLIs[ 0 ].intents()
-    count = 0
-    while count <= 5:
-        installedCheck = True
-        try:
-            for intent in json.loads( intents ):
-                state = intent.get( 'state', None )
-                if "INSTALLED" not in state:
-                    installedCheck = False
-        except ( ValueError, TypeError ):
-            main.log.exception( "Error parsing intents" )
-        if installedCheck:
-            break
-        count += 1
-    utilities.assert_equals( expect=True,
-                             actual=installedCheck ,
-                             onpass="All Intents in installed state",
-                             onfail="Not all Intents in installed state" )
-    """
+    checkIntentState( main )
+
     main.step( "Check connectivity" )
     connectivityCheck = True
     hosts = int( main.params[ 'vpls' ][ 'hosts' ] )
@@ -147,3 +126,90 @@
                              actual=connectivityCheck,
                              onpass="Connectivity is as expected",
                              onfail="Connectivity is not as expected" )
+
+
+
+# TODO: if encapsulation is set, look for that
+# TODO: can we look at the intent keys?
+
+def checkIntentState( main , bl=[] ):
+    # Print the intent states
+    intents = main.CLIs[ 0 ].intents()
+    count = 0
+    while count <= 5:
+        installedCheck = True
+        try:
+            i = 1
+            for intent in json.loads( intents ):
+                state = intent.get( 'state', None )
+                if "INSTALLED" not in state or ( "WITHDRAWN" not in state and "h" + str( i ) in bl ):
+                    installedCheck = False
+                i += 1
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing intents" )
+        if installedCheck:
+            break
+        count += 1
+    return installedCheck
+
+
+def getVplsHashtable( main, bl=[] ):
+    """
+    Returns a hashtable of vpls to hosts
+    """
+    result = {}
+    vplsConfig = main.vplsConfig
+    for v in vplsConfig:
+        interfaces = v[ 'interfaces' ][:]
+        for i in bl:
+            if i in interfaces:
+                interfaces.remove( i )
+        result[ v[ 'name' ] ] = interfaces
+    return result
+
+
+def testConnectivityVpls( main, blacklist=[], isNodeUp=True ):
+
+    # Can't do intent check when onos node is stopped/killed yet
+    if isNodeUp:
+        main.step( "Check intent states" )
+        intentsCheck = utilities.retry( f=checkIntentState,
+                                        retValue=False,
+                                        args=( main, blacklist ),
+                                        sleep=main.timeSleep,
+                                        attempts=main.numAttempts )
+
+        utilities.assert_equals( expect=True,
+                                 actual=intentsCheck,
+                                 onpass="All Intents in installed state",
+                                 onfail="Not all Intents in installed state" )
+
+    main.step( "Testing connectivity..." )
+
+    vplsHashtable = getVplsHashtable( main, blacklist )
+    main.log.debug( "vplsHashtable: " + str( vplsHashtable ) )
+    result = True
+    for key in vplsHashtable:
+        pingResult = utilities.retry( f=main.Mininet1.pingallHosts,
+                                      retValue=False,
+                                      args=( vplsHashtable[ key ], ),
+                                      sleep=main.timeSleep,
+                                      attempts=main.numAttempts )
+        result = result and pingResult
+
+    utilities.assert_equals( expect=main.TRUE, actual=result,
+                             onpass="Connectivity succeeded.",
+                             onfail="Connectivity failed." )
+    return result
+
+
+def compareApps( main ):
+    result = True
+    first = None
+    for cli in main.CLIs:
+        currentApps = cli.apps( summary=True, active=True )
+        if not result:
+            first = currentApps
+        else:
+            result = result and ( currentApps == first )
+    return result