Merge "Add sleep before starting dhclient command for TrellisHost"
diff --git a/TestON/bin/cli.py b/TestON/bin/cli.py
index 8e48311..263d4b4 100755
--- a/TestON/bin/cli.py
+++ b/TestON/bin/cli.py
@@ -263,10 +263,16 @@
             while index < len( args ):
                 option = args[ index ]
                 if index > 0:
-                    if re.match( "--params", option, flags=0 ):
+                    if re.match( "--params-file", option, flags=0 ):
+                        # The more specific match must be before --params
+                        options[ 'paramsFile' ] = args[ index + 1 ]
+                    elif re.match( "--topo-file", option, flags=0 ):
+                        options[ 'topoFile' ] = args[ index + 1 ]
+                    elif re.match( "--params", option, flags=0 ):
                         # check if there is a params
                         options[ 'params' ].append( args[ index + 1 ] )
-                    elif re.match( "logdir|mail|example|testdir|testcases|onoscell", option, flags = 0 ):
+                    elif re.match( "logdir|mail|example|testdir|testcases|onoscell",
+                                   option, flags=0 ):
                         options[ option ] = args[ index + 1 ]
                         options = self.testcasesInRange( index + 1, option, args, options )
                     index += 2
@@ -292,6 +298,8 @@
         options[ 'onoscell' ] = None
         # init params as a empty list
         options[ 'params' ] = []
+        options[ 'paramsFile' ] = None
+        options[ 'topoFile' ] = None
         return options
 
     def testcasesInRange( self, index, option, args, options ):
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index dc2b2b2..551bacc 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -43,9 +43,9 @@
         for component in main.componentDictionary.keys():
             logmsg = logmsg + "\n\t" + component + " Session Log : " + main.logdir + "/" + component + ".session" + ""
 
-        logmsg = logmsg + "\n\tTest Script :" + path + "Tests/" + main.TEST + ".py" + ""
-        logmsg = logmsg + "\n\tTest Params : " + path + "Tests/" + main.TEST + ".params" + ""
-        logmsg = logmsg + "\n\tTopology : " + path + "Tests/" + main.TEST + ".topo" + ""
+        logmsg = logmsg + "\n\tTest Script : " + main.testFile + ""
+        logmsg = logmsg + "\n\tTest Params : " + main.testDir + "/" +  main.paramsFile + ""
+        logmsg = logmsg + "\n\tTopology : " + main.testDir + "/" + main.topoFile + ""
         logmsg = logmsg + "\n" + " " * 30 + "+" + "-" * 18 + "+" + "\n" + "-" * 27 + "  { Script Exec Params }  " + "-" * 27 + "\n" + " " * 30 + "+" + "-" * 18 + "+\n"
         values = "\n\t" + str( main.params )
         values = re.sub( ",", "\n\t", values )
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index 6794564..19449f4 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -90,6 +90,7 @@
         self.test_target = None
         self.lastcommand = None
         self.testDir = tests_path
+        self.testsRoot = tests_path
         self.configFile = config_path + "teston.cfg"
         self.parsingClass = "xmlparser"
         self.parserPath = core_path + "/xmlparser"
@@ -935,6 +936,7 @@
                         main.classPath = directory[ index: ].replace( '/', '.' ) + "." + main.TEST
                         break
     openspeakfile = directory + "/" + main.TEST + ".ospk"
+    main.testDir = directory
     main.testFile = directory + "/" + main.TEST + ".py"
     if os.path.exists( openspeakfile ):
         # Openspeak file found, compiling to python
@@ -962,8 +964,10 @@
     testClass = getattr( testModule, main.TEST )
     main.testObject = testClass()
     load_parser()
-    main.params = main.parser.parseParams( main.classPath )
-    main.topology = main.parser.parseTopology( main.classPath )
+    main.paramsFile = main.TEST + ".params" if options.paramsFile is None else options.paramsFile
+    main.topoFile = main.TEST + ".topo" if options.topoFile is None else options.topoFile
+    main.params = main.parser.parseFile( main.testDir + "/" + main.paramsFile )
+    main.topology = main.parser.parseFile( main.testDir + "/" + main.topoFile )
 
 def verifyParams( options ):
     try:
@@ -1039,8 +1043,7 @@
                                             -1 )
                 parsingClass = getattr( parsingModule, parsingClass )
                 main.parser = parsingClass()
-                if hasattr( main.parser, "parseParams" ) and\
-                   hasattr( main.parser, "parseTopology" ) and\
+                if hasattr( main.parser, "parseFile" ) and\
                    hasattr( main.parser, "parse" ):
                     pass
                 else:
@@ -1072,8 +1075,7 @@
                                     -1 )
         parsingClass = getattr( parsingModule, parsingClass )
         main.parser = parsingClass()
-        if hasattr( main.parser, "parseParams" ) and\
-           hasattr( main.parser, "parseTopology" ) and\
+        if hasattr( main.parser, "parseFile" ) and\
            hasattr( main.parser, "parse" ):
             pass
         else:
diff --git a/TestON/core/xmlparser.py b/TestON/core/xmlparser.py
index 12a3f61..d7af564 100644
--- a/TestON/core/xmlparser.py
+++ b/TestON/core/xmlparser.py
@@ -27,6 +27,7 @@
 
 import xmldict
 import re
+import os.path
 
 class xmlparser:
 
@@ -49,23 +50,10 @@
         else:
             print "File name is not correct"
 
-    def parseParams( self, paramsPath ):
+    def parseFile( self, fileName ):
         '''
-         It will take the params file path and will return the params dictionary
+         It will take a file path of an xml file and return the contents as a dictionary
         '''
-        paramsPath = re.sub( "\.", "/", paramsPath )
-        paramsPath = re.sub( "tests|examples", "", paramsPath )
-        params = self.parse( main.tests_path + paramsPath + ".params" )
-        paramsAsString = str( params )
-        return eval( paramsAsString )
-
-    def parseTopology( self, topologyPath ):
-        '''
-          It will take topology file path and will return topology dictionary
-        '''
-        topologyPath = re.sub( "\.", "/", topologyPath )
-        topologyPath = re.sub( "tests|examples", "", topologyPath )
-        topology = self.parse( main.tests_path + topologyPath + ".topo" )
-        topoAsString = str( topology )
-        return eval( topoAsString )
+        contents = self.parse( fileName )
+        return eval( str( contents ) )
 
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 6fd807d..c273d43 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -478,10 +478,20 @@
                         isReachable = main.FALSE
                         failedPings += 1
                 pingResponse += "\n"
+                if not isReachable:
+                    main.log.warn( "Cannot ping between {} and {}".format( host, temp ) )
             main.log.info( pingResponse + "Failed pings: " + str( failedPings ) )
             return isReachable
         except pexpect.TIMEOUT:
             main.log.exception( self.name + ": TIMEOUT exception" )
+            response = self.handle.before
+            # NOTE: Send ctrl-c to make sure command is stopped
+            self.handle.sendline( "\x03" )
+            self.handle.expect( "Interrupt" )
+            response += self.handle.before + self.handle.after
+            self.handle.expect( "mininet>" )
+            response += self.handle.before + self.handle.after
+            main.log.debug( response )
             return main.FALSE
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
@@ -536,11 +546,21 @@
                         isReachable = main.FALSE
                         failedPingsTotal += 1
                 pingResponse += "\n"
+                if not isReachable:
+                    main.log.warn( "Cannot ping between {} and {}".format( host, temp ) )
             main.log.info( pingResponse + "Failed pings: " + str( failedPingsTotal ) )
             return isReachable
 
         except pexpect.TIMEOUT:
             main.log.exception( self.name + ": TIMEOUT exception" )
+            response = self.handle.before
+            # NOTE: Send ctrl-c to make sure command is stopped
+            self.handle.sendline( "\x03" )
+            self.handle.expect( "Interrupt" )
+            response += self.handle.before + self.handle.after
+            self.handle.expect( "mininet>" )
+            response += self.handle.before + self.handle.after
+            main.log.debug( response )
             return main.FALSE
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
@@ -596,6 +616,14 @@
 
         except pexpect.TIMEOUT:
             main.log.exception( self.name + ": TIMEOUT exception" )
+            response = self.handle.before
+            # NOTE: Send ctrl-c to make sure command is stopped
+            self.handle.sendline( "\x03" )
+            self.handle.expect( "Interrupt" )
+            response += self.handle.before + self.handle.after
+            self.handle.expect( "mininet>" )
+            response += self.handle.before + self.handle.after
+            main.log.debug( response )
             return main.FALSE
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
@@ -686,7 +714,6 @@
                     self.name +
                     ": PACKET LOST, HOST IS NOT REACHABLE" )
                 return main.FALSE
-
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
             main.log.error( self.name + ":     " + self.handle.before )
@@ -736,6 +763,14 @@
                     isReachable = main.FALSE
         except pexpect.TIMEOUT:
             main.log.exception( self.name + ": TIMEOUT exception" )
+            response = self.handle.before
+            # NOTE: Send ctrl-c to make sure command is stopped
+            self.handle.sendline( "\x03" )
+            self.handle.expect( "Interrupt" )
+            response += self.handle.before + self.handle.after
+            self.handle.expect( "mininet>" )
+            response += self.handle.before + self.handle.after
+            main.log.debug( response )
             isReachable = main.FALSE
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
@@ -1224,6 +1259,8 @@
             else:
                 pattern = "inet6\saddr:\s([\w,:]*)/\d+\sScope:Global"
             ipAddressSearch = re.search( pattern, response )
+            if not ipAddressSearch:
+                return None
             main.log.info(
                 self.name +
                 ": IP-Address of Host " +
diff --git a/TestON/tests/CHOTestMonkey/CHOTestMonkey.py b/TestON/tests/CHOTestMonkey/CHOTestMonkey.py
index 3efb3cc..3e02799 100644
--- a/TestON/tests/CHOTestMonkey/CHOTestMonkey.py
+++ b/TestON/tests/CHOTestMonkey/CHOTestMonkey.py
@@ -173,7 +173,7 @@
         main.step( "Start Mininet topology" )
         newTopo = main.params[ 'TOPO' ][ main.topoIndex ][ 'fileName' ]
         mininetDir = main.Mininet1.home + "/custom/"
-        topoPath = main.testDir + "/" + main.TEST + "/dependencies/topologies/" + newTopo
+        topoPath = main.testDir + "/dependencies/topologies/" + newTopo
         main.ONOSbench.secureCopy( main.Mininet1.user_name, main.Mininet1.ip_address, topoPath, mininetDir, direction="to" )
         topoPath = mininetDir + newTopo
         startStatus = main.Mininet1.startNet( topoFile=topoPath )
diff --git a/TestON/tests/PLAT/PLATdockertest/PLATdockertest.py b/TestON/tests/PLAT/PLATdockertest/PLATdockertest.py
index d871fae..bc4f3c3 100644
--- a/TestON/tests/PLAT/PLATdockertest/PLATdockertest.py
+++ b/TestON/tests/PLAT/PLATdockertest/PLATdockertest.py
@@ -91,7 +91,7 @@
         NODElist = main.params[ "SCALE" ][ "nodelist" ].split( ',' )
         main.log.info( "onos container names are: " + ",".join( NODElist ) )
         IPlist = list()
-        main.testOnDirectory = re.sub( "(/tests)$", "", main.testDir )
+        main.testOnDirectory = re.sub( "(/tests)$", "", main.testsRoot )
         CTIDlist = list()
 
         main.log.info( "Check docker status, it not running, try restart it" )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index f808d4e..07f1260 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -558,4 +558,183 @@
                                staticRouteConfigure=True,
                                switchFailure=True )
 
+    def CASE301( self, main ):
+        """
+        Kill and recover ONOS nodes
+        Ping between all ipv4 hosts in the topology.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=301,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=0,
+                               countFlowsGroups=False,
+                               description="Test node failures with IPv4 hosts",
+                               nodeFailure=True )
+
+    def CASE302( self, main ):
+        """
+        Kill and recover ONOS nodes
+        Ping between all ipv6 hosts in the topology.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=302,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=0,
+                               ipv6=1,
+                               countFlowsGroups=False,
+                               description="Test node failures with IPv6 hosts",
+                               nodeFailure=True )
+
+    def CASE303( self, main ):
+        """
+        Kill and recover ONOS nodes
+        Ping between all ipv4 and ipv6 hosts in the topology.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=303,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=1,
+                               countFlowsGroups=False,
+                               description="Test node failures with IPv4 and IPv6 hosts",
+                               nodeFailure=True )
+
+    def CASE304( self, main ):
+        """
+        Kill and recover ONOS nodes
+        Ping between all ipv4 hosts in the topology and check connectivity to external ipv4 hosts
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=304,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=0,
+                               description="Test node failures with IPv4 hosts (including external hosts)",
+                               checkExternalHost=True,
+                               nodeFailure=True )
+
+    def CASE305( self, main ):
+        """
+        Kill and recover ONOS nodes
+        Ping between all ipv6 hosts in the topology and check connectivity to external ipv6 hosts
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=305,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=0,
+                               ipv6=1,
+                               description="Test node failures with IPv6 hosts (including external hosts)",
+                               checkExternalHost=True,
+                               nodeFailure=True )
+
+    def CASE306( self, main ):
+        """
+        Kill and recover ONOS nodes
+        Ping between all ipv4 and ipv6 hosts in the topology and check connectivity to external ipv4 and ipv6 hosts
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=306,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=1,
+                               description="Test node failures with IPv4 and IPv6 hosts (including external hosts)",
+                               checkExternalHost=True,
+                               nodeFailure=True )
+
+    def CASE307( self, main ):
+        """
+        Kill and recover ONOS nodes
+        Ping between ipv4 hosts and an external host that is not configured in
+        external router config, but reachable through the use of route-add command.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=307,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=0,
+                               description="Test node failures with IPv4 hosts (including external host configured with route-add command)",
+                               checkExternalHost=False,
+                               countFlowsGroups=False,
+                               staticRouteConfigure=True,
+                               nodeFailure=True )
+
+    def CASE308( self, main ):
+        """
+        Kill and recover ONOS nodes
+        Ping between ipv6 hosts and an external host that is not configured in
+        external router config, but reachable through the use of route-add command.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=308,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=0,
+                               ipv6=1,
+                               description="Test node failures with IPv6 hosts (including external host configured with route-add command)",
+                               checkExternalHost=False,
+                               countFlowsGroups=False,
+                               staticRouteConfigure=True,
+                               nodeFailure=True )
+
+    def CASE309( self, main ):
+        """
+        Kill and recover ONOS nodes
+        Ping between ipv4 and pv6 hosts and external hosts that is not configured in
+        external router config, but reachable through the use of route-add command.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=309,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=1,
+                               description="Test node failures with IPv4 and IPv6 hosts (including external host configured with route-add command)",
+                               checkExternalHost=False,
+                               countFlowsGroups=False,
+                               staticRouteConfigure=True,
+                               nodeFailure=True )
 
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index 4b58d10..5707a7c 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -21,6 +21,7 @@
 
 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as run
 import time
+import json
 
 class SRRoutingTest ():
 
@@ -32,7 +33,8 @@
     @staticmethod
     def runTest( main, test_idx, onosNodes, dhcp, routers, ipv4, ipv6,
                  description, countFlowsGroups=False, checkExternalHost=False,
-                 staticRouteConfigure=False, switchFailure=False, linkFailure=False ):
+                 staticRouteConfigure=False, switchFailure=False, linkFailure=False,
+                 nodeFailure=False ):
 
         skipPackage = False
         init = False
@@ -53,7 +55,7 @@
         if staticRouteConfigure:
             main.cfgName += '_static=1'
 
-        main.resultFileName = 'CASE%02d' % test_idx
+        main.resultFileName = 'CASE%03d' % test_idx
         main.Cluster.setRunningNode( onosNodes )
 
         run.installOnos( main, skipPackage=skipPackage, cliSleep=5,
@@ -97,7 +99,7 @@
             time.sleep( 60 )
 
         # ping hosts
-        run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+        run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
 
         # check flows / groups numbers
         if countFlowsGroups:
@@ -107,11 +109,11 @@
         if switchFailure:
             for switch, expected in main.switchFailureChart.items():
                 run.killSwitch( main, switch, expected['switches_after_failure'], expected['links_after_failure'] )
-                run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+                run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
                 if countFlowsGroups:
                     run.checkFlowsGroupsFromFile(main)
                 run.recoverSwitch( main, switch, expected['switches_before_failure'], expected['links_before_failure'] )
-                run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+                run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
                 if countFlowsGroups:
                     run.checkFlowsGroupsFromFile(main)
 
@@ -124,13 +126,32 @@
                 linksAfter = info['links_after']
 
                 run.killLinkBatch( main, linksToRemove, linksAfter )
-                run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+                run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
 
                 run.restoreLinkBatch( main, linksToRemove, linksBefore )
-                run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+                run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
                 if countFlowsGroups:
                     run.checkFlowsGroupsFromFile(main)
 
+        # Test node failures
+        if nodeFailure:
+            numCtrls = len( main.Cluster.runningNodes )
+            links = len( json.loads( main.Cluster.next().links() ) )
+            switches = len( json.loads( main.Cluster.next().devices() ) )
+            for ctrl in xrange( numCtrls ):
+                run.killOnos( main, [ ctrl ], switches, links, ( numCtrls - 1 ) )
+                main.Cluster.active(0).CLI.balanceMasters()
+                run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
+                if countFlowsGroups:
+                    run.checkFlowsGroupsFromFile( main )
+
+                run.recoverOnos( main, [ ctrl ], switches, links, numCtrls )
+                main.Cluster.active(0).CLI.balanceMasters()
+                run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
+                if countFlowsGroups:
+                    run.checkFlowsGroupsFromFile( main )
+
+        # Cleanup
         if hasattr( main, 'Mininet1' ):
             run.cleanup( main )
         else:
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 0125da6..bd63a80 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -97,8 +97,8 @@
         # main.scale[ 0 ] determines the current number of ONOS controller
         if not main.apps:
             main.log.error( "App list is empty" )
-        main.log.info( "NODE COUNT = " + str( main.Cluster.numCtrls ) )
-        main.log.info( ''.join( main.Cluster.getIps() ) )
+        main.log.info( "Cluster size: " + str( main.Cluster.numCtrls ) )
+        main.log.info( "Cluster ips: " + ', '.join( main.Cluster.getIps() ) )
         main.dynamicHosts = [ 'in1', 'out1' ]
         main.testSetUp.ONOSSetUp( main.Cluster, newCell=True, cellName=main.cellName,
                                   skipPack=skipPackage,
@@ -280,7 +280,7 @@
     @staticmethod
     def checkFlows( main, minFlowCount, tag="", dumpflows=True, sleep=10 ):
         main.step(
-                " Check whether the flow count is bigger than %s" % minFlowCount )
+                "Check whether the flow count is bigger than %s" % minFlowCount )
         if tag == "":
             tag = 'CASE%d' % main.CurrentTestCaseNumber
         count = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowCount,
@@ -638,7 +638,7 @@
         switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
         Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
         """
-        main.step( "Killing ONOS instance" )
+        main.step( "Killing ONOS instances with index(es): {}".format( nodes ) )
 
         for i in nodes:
             killResult = main.ONOSbench.onosDie( main.Cluster.runningNodes[ i ].ipAddress )
@@ -685,7 +685,7 @@
         switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
         Recover an ONOS instance and verify the ONOS cluster can see the proper change
         """
-        main.step( "Recovering ONOS instance" )
+        main.step( "Recovering ONOS instances with index(es): {}".format( nodes ) )
         [ main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) for i in nodes ]
         for i in nodes:
             isUp = main.ONOSbench.isup( main.Cluster.runningNodes[ i ].ipAddress )
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/trellislib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/trellislib.py
index a998b4e..b87ec9f 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/trellislib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/trellislib.py
@@ -47,26 +47,40 @@
 
     def config(self, **kwargs):
         super(DualHomedRoutedHost, self).config(**kwargs)
-        intf0 = self.intfs[0].name
-        intf1 = self.intfs[1].name
-        self.bond0 = "%s-bond0" % self.name
-        self.cmd('modprobe bonding')
-        self.cmd('ip link add %s type bond' % self.bond0)
-        self.cmd('ip link set %s down' % intf0)
-        self.cmd('ip link set %s down' % intf1)
-        self.cmd('ip link set %s master %s' % (intf0, self.bond0))
-        self.cmd('ip link set %s master %s' % (intf1, self.bond0))
-        self.cmd('ip addr flush dev %s' % intf0)
-        self.cmd('ip addr flush dev %s' % intf1)
-        self.cmd('ip link set %s up' % self.bond0)
+        self.bondIntfs( self.intfs[0], self.intfs[1] )
 
         for ip in self.ips:
             self.cmd('ip addr add %s dev %s' % (ip, self.bond0))
 
         self.cmd('ip route add default via %s' % self.gateway)
-        default_intf = self.defaultIntf()
-        default_intf.name = self.bond0
-        self.nameToIntf[self.bond0] = default_intf
+
+    def bondIntfs( self, intf1, intf2, bondedName="bond0" ):
+        '''
+        Bond two interfaces together
+        intf1 - the first interface to bond
+        intf2 - the second interface to bond
+        bondedName - the prefix of the new interface name
+        '''
+        # Setup bonded interface
+        # TODO: support multiple bonded interfaces. Maybe just changed self.bond0 to a list of bonded intf names?
+        self.bond0 = "%s-%s" % ( self.name, bondedName )
+        self.cmd('modprobe bonding')
+        self.cmd('ip link add %s type bond' % self.bond0)
+        self.cmd('ip link set %s down' % intf1.name)
+        self.cmd('ip link set %s down' % intf2.name)
+        self.cmd('ip link set %s master %s' % (intf1.name, self.bond0))
+        self.cmd('ip link set %s master %s' % (intf2.name, self.bond0))
+        self.cmd('ip addr flush dev %s' % intf1.name)
+        self.cmd('ip addr flush dev %s' % intf2.name)
+        self.cmd('ip link set %s up' % self.bond0)
+        # NOTE: Issues with bonded intfs in mn data structures. Either only show bonded intf
+        #       or create a custom class to handle bonded infs??
+        lowestIntf = min( [ intf1, intf2 ] )
+        highestIntf = max( [ intf1, intf2 ] )
+        lowestIntf.name = self.bond0
+        self.nameToIntf[self.bond0] = lowestIntf
+        del self.intfs[ self.ports[ highestIntf ] ]
+        del self.ports[ highestIntf ]
 
     def terminate(self, **kwargs):
         self.cmd('ip link set %s down' % self.bond0)
@@ -232,19 +246,36 @@
 
     def config(self, **kwargs):
         super(DualHomedDhcpClient, self).config(**kwargs)
-        intf0 = self.intfs[0].name
-        intf1 = self.intfs[1].name
-        self.bond0 = "%s-bond0" % self.name
+        self.bondIntfs( self.intfs[0], self.intfs[1] )
+        self.cmd('dhclient -q -4 -nw -pf %s %s' % (self.pidFile, self.bond0))
+
+    def bondIntfs( self, intf1, intf2, bondedName="bond0" ):
+        '''
+        Bond two interfaces together
+        intf1 - the first interface to bond
+        intf2 - the second interface to bond
+        bondedName - the prefix of the new interface name
+        '''
+        # Setup bonded interface
+        # TODO: support multiple bonded interfaces. Maybe just changed self.bond0 to a list of bonded intf names?
+        self.bond0 = "%s-%s" % ( self.name, bondedName )
         self.cmd('modprobe bonding')
         self.cmd('ip link add %s type bond' % self.bond0)
-        self.cmd('ip link set %s down' % intf0)
-        self.cmd('ip link set %s down' % intf1)
-        self.cmd('ip link set %s master %s' % (intf0, self.bond0))
-        self.cmd('ip link set %s master %s' % (intf1, self.bond0))
-        self.cmd('ip addr flush dev %s' % intf0)
-        self.cmd('ip addr flush dev %s' % intf1)
+        self.cmd('ip link set %s down' % intf1.name)
+        self.cmd('ip link set %s down' % intf2.name)
+        self.cmd('ip link set %s master %s' % (intf1.name, self.bond0))
+        self.cmd('ip link set %s master %s' % (intf2.name, self.bond0))
+        self.cmd('ip addr flush dev %s' % intf1.name)
+        self.cmd('ip addr flush dev %s' % intf2.name)
         self.cmd('ip link set %s up' % self.bond0)
-        self.cmd('dhclient -q -4 -nw -pf %s %s' % (self.pidFile, self.bond0))
+        # NOTE: Issues with bonded intfs in mn data structures. Either only show bonded intf
+        #       or create a custom class to handle bonded infs??
+        lowestIntf = min( [ intf1, intf2 ] )
+        highestIntf = max( [ intf1, intf2 ] )
+        lowestIntf.name = self.bond0
+        self.nameToIntf[self.bond0] = lowestIntf
+        del self.intfs[ self.ports[ highestIntf ] ]
+        del self.ports[ highestIntf ]
 
     def terminate(self, **kwargs):
         self.cmd('ip link set %s down' % self.bond0)
@@ -279,22 +310,7 @@
         super(TrellisHost, self).config(**kwargs)
 
         if self.dualHomed:
-            # Setup bond0 interface
-            intf0 = self.intfs[0].name
-            intf1 = self.intfs[1].name
-            self.bond0 = "%s-bond0" % self.name
-            self.cmd('modprobe bonding')
-            self.cmd('ip link add %s type bond' % self.bond0)
-            self.cmd('ip link set %s down' % intf0)
-            self.cmd('ip link set %s down' % intf1)
-            self.cmd('ip link set %s master %s' % (intf0, self.bond0))
-            self.cmd('ip link set %s master %s' % (intf1, self.bond0))
-            self.cmd('ip addr flush dev %s' % intf0)
-            self.cmd('ip addr flush dev %s' % intf1)
-            self.cmd('ip link set %s up' % self.bond0)
-            defaultIntf = self.defaultIntf()
-            defaultIntf.name = self.bond0
-            self.nameToIntf[self.bond0] = defaultIntf
+            self.bondIntfs( self.intfs[0], self.intfs[1] )
 
         self.cmd('ip %s addr flush dev %s' % ("-4" if self.ipv6 else "", self.defaultIntf()))
 
@@ -323,6 +339,34 @@
                 self.cmd('touch %s' % self.leasesFile)
                 self.cmd('%s -q -%s -pf %s -cf %s %s' % (self.binFile, 6 if self.ipv6 else 4, self.pidFile, self.configFile, self.defaultIntf()))
 
+    def bondIntfs( self, intf1, intf2, bondedName="bond0" ):
+        '''
+        Bond two interfaces together
+        intf1 - the first interface to bond
+        intf2 - the second interface to bond
+        bondedName - the prefix of the new interface name
+        '''
+        # Setup bonded interface
+        # TODO: support multiple bonded interfaces. Maybe just changed self.bond0 to a list of bonded intf names?
+        self.bond0 = "%s-%s" % ( self.name, bondedName )
+        self.cmd('modprobe bonding')
+        self.cmd('ip link add %s type bond' % self.bond0)
+        self.cmd('ip link set %s down' % intf1.name)
+        self.cmd('ip link set %s down' % intf2.name)
+        self.cmd('ip link set %s master %s' % (intf1.name, self.bond0))
+        self.cmd('ip link set %s master %s' % (intf2.name, self.bond0))
+        self.cmd('ip addr flush dev %s' % intf1.name)
+        self.cmd('ip addr flush dev %s' % intf2.name)
+        self.cmd('ip link set %s up' % self.bond0)
+        # NOTE: Issues with bonded intfs in mn data structures. Either only show bonded intf
+        #       or create a custom class to handle bonded infs??
+        lowestIntf = min( [ intf1, intf2 ] )
+        highestIntf = max( [ intf1, intf2 ] )
+        lowestIntf.name = self.bond0
+        self.nameToIntf[self.bond0] = lowestIntf
+        del self.intfs[ self.ports[ highestIntf ] ]
+        del self.ports[ highestIntf ]
+
     def terminate(self, **kwargs):
         if self.vlan:
             self.cmd('ip link remove link %s' % self.vlanIntf)
diff --git a/TestON/tests/USECASE/USECASE_SdnipFunction/USECASE_SdnipFunction.py b/TestON/tests/USECASE/USECASE_SdnipFunction/USECASE_SdnipFunction.py
index a45bdef..573f16d 100644
--- a/TestON/tests/USECASE/USECASE_SdnipFunction/USECASE_SdnipFunction.py
+++ b/TestON/tests/USECASE/USECASE_SdnipFunction/USECASE_SdnipFunction.py
@@ -33,7 +33,7 @@
         """
         import os
         main.case( "Setup the Mininet testbed" )
-        main.dependencyPath = main.testDir + \
+        main.dependencyPath = main.testsRoot + \
                               main.params[ 'DEPENDENCY' ][ 'path' ]
         main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
 
diff --git a/TestON/tests/USECASE/USECASE_SdnipFunctionCluster/USECASE_SdnipFunctionCluster.py b/TestON/tests/USECASE/USECASE_SdnipFunctionCluster/USECASE_SdnipFunctionCluster.py
index 499df34..b984b1d 100644
--- a/TestON/tests/USECASE/USECASE_SdnipFunctionCluster/USECASE_SdnipFunctionCluster.py
+++ b/TestON/tests/USECASE/USECASE_SdnipFunctionCluster/USECASE_SdnipFunctionCluster.py
@@ -33,7 +33,7 @@
         """
         import imp
         main.case( "Setup the Mininet testbed" )
-        main.dependencyPath = main.testDir + \
+        main.dependencyPath = main.testsRoot + \
                               main.params[ 'DEPENDENCY' ][ 'path' ]
         main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
 
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index ca10d3a..ee339be 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -46,7 +46,7 @@
         except ( NameError, AttributeError ):
             main.Cluster = Cluster( main.ONOScell.nodes )
         main.ONOSbench = main.Cluster.controllers[ 0 ].Bench
-        main.testOnDirectory = re.sub( "(/tests)$", "", main.testDir )
+        main.testOnDirectory = re.sub( "(/tests)$", "", main.testsRoot )
 
     def gitPulling( self, includeCaseDesc=True ):
         """