Add Basic Connectivity tests for paire leaf pod

Change-Id: Ie2c7529629893462e9567700354368380188ef8b
diff --git a/TestON/drivers/common/cli/hostdriver.py b/TestON/drivers/common/cli/hostdriver.py
index e51c5d6..d0cf559 100644
--- a/TestON/drivers/common/cli/hostdriver.py
+++ b/TestON/drivers/common/cli/hostdriver.py
@@ -224,11 +224,21 @@
         """
         cmd = "ifconfig %s" % iface if iface else ""
         response = self.command( cmd )
-        pattern = ''
-        if proto == 'IPV4':
-            pattern = "inet\s(\d+\.\d+\.\d+\.\d+)\s\snetmask"
+        if "Command 'ifconfig' not found" in response:
+            # ip a show dev
+            cmd = "ip addr show %s" % iface if iface else ""
+            response = self.command( cmd )
+            pattern = ''
+            if proto == 'IPV4':
+                pattern = "inet\s(\d+\.\d+\.\d+\.\d+)/\d+"
+            else:
+                pattern = "inet6\s([\w,:]*)/\d+"
         else:
-            pattern = "inet6\s([\w,:]*)/\d+\s\sprefixlen"
+            pattern = ''
+            if proto == 'IPV4':
+                pattern = "inet\s(\d+\.\d+\.\d+\.\d+)\s\snetmask"
+            else:
+                pattern = "inet6\s([\w,:]*)/\d+\s\sprefixlen"
         ipAddressSearch = re.search( pattern, response )
         if not ipAddressSearch:
             return None
@@ -335,9 +345,15 @@
             command = "ifconfig"
             main.log.info( self.name + ": Sending: " + command )
             self.handle.sendline( command )
-            i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT ],
+            i = self.handle.expect( [ "Command 'ifconfig' not found", self.prompt, pexpect.TIMEOUT ],
                                     timeout=wait + 5 )
-            if i == 1:
+            if i == 0:
+                response = self.handle.before
+                self.handle.expect( [ self.prompt, pexpect.TIMEOUT ] )
+                response += str( self.handle.before )
+                main.log.error( self.name + ": Error running ifconfig: %s " % response )
+                return response
+            if i == 2:
                 main.log.error(
                     self.name +
                     ": timeout when waiting for response" )
diff --git a/TestON/drivers/common/cli/networkdriver.py b/TestON/drivers/common/cli/networkdriver.py
index 6d46c1e..6b02e09 100755
--- a/TestON/drivers/common/cli/networkdriver.py
+++ b/TestON/drivers/common/cli/networkdriver.py
@@ -786,23 +786,12 @@
         """
         Returns IP address of the host
         """
-        cmd = "ifconfig %s" % iface if iface else ""
-        response = self.runCmdOnHost( host, cmd )
-        pattern = ''
-        if proto == 'IPV4':
-            pattern = "inet\s(\d+\.\d+\.\d+\.\d+)\s\snetmask"
+        hostComponent = self.hosts[ host ]
+        if hostComponent:
+            return hostComponent.getIPAddress( iface=iface, proto=proto )
         else:
-            pattern = "inet6\s([\w,:]*)/\d+\s\sprefixlen"
-        ipAddressSearch = re.search( pattern, response )
-        if not ipAddressSearch:
+            main.log.warn( self.name + ": Could not find host with short name '%s'" % host )
             return None
-        main.log.info(
-            self.name +
-            ": IP-Address of Host " +
-            host +
-            " is " +
-            ipAddressSearch.group( 1 ) )
-        return ipAddressSearch.group( 1 )
 
     def getLinkRandom( self, timeout=60, nonCut=True, excludeNodes=[], skipLinks=[] ):
         """
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/dependencies/SRClusterRestartFuncs.py b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/dependencies/SRClusterRestartFuncs.py
index be727a3..cf559be 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/dependencies/SRClusterRestartFuncs.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/dependencies/SRClusterRestartFuncs.py
@@ -54,7 +54,7 @@
             switch = '{}'.format( self.topo[ topology ][ 'spines' ] + self.topo[ topology ][ 'leaves' ] )
             link = '{}'.format( ( self.topo[ topology ][ 'spines' ] + self.topo[ topology ][ 'leaves' ] ) * self.topo[ topology ][ 'spines' ] )
             run.killOnos( main, killList, switch, link, '0' )
-            run.pingAll( main, caseTitle, dumpflows=False )
+            run.pingAll( main, caseTitle, dumpFlows=False )
             run.recoverOnos( main, killList, switch, link, '{}'.format( numNodes ) )
             run.checkFlows( main, minFlowCount=minFlow, tag=caseTitle )
             run.pingAll( main, caseTitle )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDynamic/dependencies/SRDynamicFuncs.py b/TestON/tests/USECASE/SegmentRouting/SRDynamic/dependencies/SRDynamicFuncs.py
index 7c83f80..ad6839e 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRDynamic/dependencies/SRDynamicFuncs.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRDynamic/dependencies/SRDynamicFuncs.py
@@ -51,9 +51,9 @@
                               args=self.topo[ topology ][ 'mininetArgs' ] )
             # pre-configured routing and bridging test
             run.checkFlows( main, minFlowCount=minBeforeFlow )
-            run.pingAll( main, dumpflows=False )
+            run.pingAll( main, dumpFlows=False )
             run.addHostCfg( main )
-            run.checkFlows( main, minFlowCount=minAfterFlow, dumpflows=False )
+            run.checkFlows( main, minFlowCount=minAfterFlow, dumpFlows=False )
             run.pingAll( main )
             if killOnosAndDeleteCfg:
                 switch = self.topo[ topology ][ 'spines' ] + self.topo[ topology ][ 'leaves' ]
@@ -73,5 +73,5 @@
     def killAndDelete( self, main, caseNum, numNodes, minBeforeFlow, switch, link ):
         run.killOnos( main, [ 0 ], '{}'.format( switch ), '{}'.format( link ), '{}'.format( numNodes - 1 ) )
         run.delHostCfg( main )
-        run.checkFlows( main, minFlowCount=minBeforeFlow, dumpflows=False )
+        run.checkFlows( main, minFlowCount=minBeforeFlow, dumpFlows=False )
         run.pingAll( main, "CASE{}_After".format( caseNum ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/dependencies/SRDynamicConfTest.py b/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/dependencies/SRDynamicConfTest.py
index 70b4e6b..f58e4f8 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/dependencies/SRDynamicConfTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/dependencies/SRDynamicConfTest.py
@@ -137,7 +137,7 @@
 
             # minFlowCountPerLeaf = 13 +  [# of ports] * 5 + [# of hosts] * 2 + [# of vlan ids]
             minFlowCountPerLeaf = 13 + ( fanout + self.topo[ topology ][ 'spines' ]) * 5 + fanout * 2 + len( set( vlan ) )
-            run.checkFlows( main, minFlowCount=minFlowCountPerLeaf * self.topo[ topology ][ 'leaves' ], sleep=5, dumpflows=False )
+            run.checkFlows( main, minFlowCount=minFlowCountPerLeaf * self.topo[ topology ][ 'leaves' ], sleep=5, dumpFlows=False )
             # Check connectivity before changing interface configuration
             run.pingAll( main, '%s_Before' % TAG, retryAttempts=2 )
 
@@ -370,7 +370,7 @@
                 main.log.debug( "Ping chart is not changed" )
 
             # Check connectivity after changing interface configuration
-            run.checkFlows( main, minFlowCount=minFlowCountPerLeaf * self.topo[ topology ][ 'leaves' ], sleep=5, dumpflows=False )
+            run.checkFlows( main, minFlowCount=minFlowCountPerLeaf * self.topo[ topology ][ 'leaves' ], sleep=5, dumpFlows=False )
             run.pingAll( main, '%s_After' % TAG, retryAttempts=2 )
 
         except Exception as e:
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
new file mode 100644
index 0000000..0834030
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
@@ -0,0 +1,92 @@
+<PARAMS>
+    <testcases>1,2</testcases>
+
+    <GRAPH>
+        <nodeCluster>pairedleaves</nodeCluster>
+        <builds>20</builds>
+        <jobName>SRpairedLeaves</jobName>
+        <branch>master</branch>
+    </GRAPH>
+
+    <SCALE>
+        <size>3</size>
+        <max>3</max>
+    </SCALE>
+
+    <DEPENDENCY>
+        <useCommonConf>False</useCommonConf>
+        <useCommonTopo>True</useCommonTopo>
+        <useBmv2>True</useBmv2>
+        <bmv2SwitchType>stratum</bmv2SwitchType>
+        <switchPrefix></switchPrefix>
+        <stratumRoot>~/stratum</stratumRoot>
+        <topology>trellis_fabric.py</topology>
+        <lib>routinglib.py,trellislib.py,stratum.py</lib>
+    </DEPENDENCY>
+
+    <persistent_setup>True</persistent_setup>
+
+    <kubernetes>
+        <appName>onos-classic</appName>
+        <namespace>tost</namespace>
+    </kubernetes>
+
+    <PERF>
+        <traffic_host>Compute1 Compute2</traffic_host>
+        <pcap_host>Compute3</pcap_host>
+        <pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
+        <iterations>1</iterations>
+        <topo>
+            <leaf1>
+                <ports>176 180 184 188</ports>
+                <note>eNB</note>
+            </leaf1>
+            <leaf2>
+                <ports>260 268 276 284</ports>
+                <note>upstream</note>
+            </leaf2>
+        </topo>
+    </PERF>
+    <ONOS_Logging>
+        <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
+        <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
+    </ONOS_Logging>
+    <ONOS_Logging_Reset>
+        <org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
+        <org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
+    </ONOS_Logging_Reset>
+
+
+    <ENV>
+        <cellName>productionCell</cellName>
+        <cellApps>drivers,fpm,lldpprovider,hostprovider,netcfghostprovider,drivers.bmv2,org.opencord.fabric-tofino,pipelines.fabric,org.stratumproject.fabric-tna,drivers.barefoot,segmentrouting,t3</cellApps>
+    </ENV>
+
+    <EXTERNAL_APPS>
+    </EXTERNAL_APPS>
+
+    <CTRL>
+        <port>6653</port>
+    </CTRL>
+
+    <timers>
+        <LinkDiscovery>12</LinkDiscovery>
+        <SwitchDiscovery>12</SwitchDiscovery>
+        <TrafficDiscovery>10</TrafficDiscovery>
+    </timers>
+
+    <SLEEP>
+        <startup>10</startup>
+    </SLEEP>
+
+    <TOPO>
+        <switchNum>2</switchNum>
+        <linkNum>2</linkNum>
+    </TOPO>
+
+    <ALARM>
+        <minPassPercent>100</minPassPercent>
+    </ALARM>
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
new file mode 100644
index 0000000..f46f0a5
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
@@ -0,0 +1,70 @@
+class SRpairedLeaves:
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        """
+        Connect to Pod
+        Check host dataplane connectivity
+        Collect logs and analyze results
+        """
+        try:
+            from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+            from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as run
+            import datetime
+            import json
+        except ImportError as e:
+            main.log.exception( "SRStagingTest not found. Exiting the test" )
+            main.cleanAndExit()
+        try:
+            main.funcs
+        except ( NameError, AttributeError ):
+            main.funcs = SRStagingTest()
+
+        try:
+            main.log.debug( "loading parser script" )
+            import tests.USECASE.SegmentRouting.SRStaging.dependencies.log_breakdown as logParser
+        except ImportError as e:
+            main.log.exception( "Error running script" )
+        descPrefix = "Host Connectivity"
+        pod = main.params['GRAPH'].get( 'nodeCluster', "hardware" )
+        main.cfgName = 'CASE001'
+        main.funcs.setupTest( main,
+                              topology='0x2',
+                              onosNodes=3,
+                              description="%s tests on the %s pod" % ( descPrefix, pod ) )
+        switches = int( main.params[ 'TOPO' ][ 'switchNum' ] )
+        links = int( main.params[ 'TOPO' ][ 'linkNum' ] )
+        hosts = [ 'h1', 'h2', 'h3', 'mgmt' ]
+        run.verifyTopology( main, switches, links, main.Cluster.numCtrls )
+        #run.loadChart( main )  # stores hosts to ping and expected results
+        #run.pingAll( main, useScapy=False )
+        run.verifyPing( main, hosts, hosts )
+        #main.funcs.cleanup( main )
+        # run.verifyTraffic
+
+    def CASE2( self, main ):
+        """
+        Connect to Pod
+        Check host to gateway connectivity
+        Collect logs and analyze results
+        """
+        try:
+            from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+            import json
+        except ImportError:
+            main.log.error( "SRStagingTest not found. Exiting the test" )
+            main.cleanAndExit()
+        try:
+            main.funcs
+        except ( NameError, AttributeError ):
+            main.funcs = SRStagingTest()
+
+        descPrefix = "Host to gateway connectivity"
+        pod = main.params['GRAPH'].get( 'nodeCluster', "hardware" )
+        main.funcs.setupTest( main,
+                              topology='0x2',
+                              onosNodes=3,
+                              description="%s tests on the %s pod" % ( descPrefix, pod ) )
+        run.pingAllFabricIntfs( main, hosts, dumpFlows=False )
+        #main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
new file mode 100644
index 0000000..40a4c6c
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
@@ -0,0 +1,188 @@
+<TOPOLOGY>
+    <COMPONENT>
+        <ONOScell>
+            <host>localhost</host>  # ONOS "bench" machine
+            <user>jenkins</user>
+            <password></password>
+            <type>OnosClusterDriver</type>
+            <connect_order>50</connect_order>
+            <jump_host></jump_host>
+            <home>~/onos</home>   # defines where onos home is on the build machine. Defaults to "~/onos/" if empty.
+            <COMPONENTS>
+                <kubeConfig>~/.kube/dev-pairedleaves-tuscon</kubeConfig>  # If set, will attempt to use this file for setting up port-forwarding
+                <useDocker>True</useDocker>  # Whether to use docker for ONOS nodes
+                <docker_prompt>\$</docker_prompt>
+                <cluster_name></cluster_name>  # Used as a prefix for cluster components. Defaults to 'ONOS'
+                <diff_clihost>True</diff_clihost> # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
+                <karaf_username>karaf</karaf_username>
+                <karaf_password>karaf</karaf_password>
+                <web_user>karaf</web_user>
+                <web_pass>karaf</web_pass>
+                <karafPrompt_username>karaf</karafPrompt_username>
+                <rest_port></rest_port>
+                <prompt></prompt>  # TODO: we technically need a few of these, one per component
+                <onos_home>~/onos/</onos_home>  # defines where onos home is on the target cell machine. Defaults to entry in "home" if empty.
+                <nodes> 3 </nodes>  # number of nodes in the cluster
+            </COMPONENTS>
+        </ONOScell>
+
+        <Leaf1>
+            <host>10.76.28.70</host>
+            <user>root</user>
+            <password>onl</password>
+            <type>StratumOSSwitchDriver</type>
+            <connect_order>12</connect_order>
+            <jump_host></jump_host>
+            <COMPONENTS>
+                <shortName>leaf1</shortName>
+                <port1></port1>
+                <link1></link1>
+                <port2></port2>
+                <link2></link2>
+                <onosConfigPath></onosConfigPath>
+                <onosConfigFile></onosConfigFile>
+            </COMPONENTS>
+        </Leaf1>
+
+        <Leaf2>
+            <host>10.76.28.71</host>
+            <user>root</user>
+            <password>onl</password>
+            <type>StratumOSSwitchDriver</type>
+            <connect_order>13</connect_order>
+            <jump_host></jump_host>
+            <COMPONENTS>
+                <shortName>leaf2</shortName>
+                <port1></port1>
+                <link1></link1>
+                <port2></port2>
+                <link2></link2>
+                <onosConfigPath></onosConfigPath>
+                <onosConfigFile></onosConfigFile>
+            </COMPONENTS>
+        </Leaf2>
+
+        <Compute1>
+            <host>10.76.28.74</host>
+            <user>jenkins</user>
+            <password></password>
+            <type>HostDriver</type>
+            <connect_order>6</connect_order>
+            <jump_host></jump_host>
+            <COMPONENTS>
+                <mac></mac>
+                <inband>false</inband>
+                <dhcp>True</dhcp>
+                <ip>10.32.11.2</ip>
+                <shortName>h1</shortName>
+                <port1></port1>
+                <link1></link1>
+                <interfaceName>pairbond</interfaceName>
+                <routes>
+                    <route1>
+                        <network></network>
+                        <netmask></netmask>
+                        <gw></gw>
+                        <interface></interface>
+                    </route1>
+                </routes>
+                <sudo_required>false</sudo_required>
+            </COMPONENTS>
+        </Compute1>
+
+        <Compute2>
+            <host>10.76.28.72</host>
+            <user>jenkins</user>
+            <password></password>
+            <type>HostDriver</type>
+            <connect_order>7</connect_order>
+            <jump_host></jump_host>
+            <COMPONENTS>
+                <mac></mac>
+                <inband>false</inband>
+                <dhcp>True</dhcp>
+                <ip>10.32.11.3</ip>
+                <shortName>h2</shortName>
+                <port1></port1>
+                <link1></link1>
+                <interfaceName>pairbond</interfaceName>
+                <routes>
+                    <route1>
+                        <network></network>
+                        <netmask></netmask>
+                        <gw></gw>
+                        <interface></interface>
+                    </route1>
+                </routes>
+                <sudo_required>false</sudo_required>
+            </COMPONENTS>
+        </Compute2>
+
+        <Compute3>
+            <host>10.76.28.68</host>
+            <user>jenkins</user>
+            <password></password>
+            <type>HostDriver</type>
+            <connect_order>8</connect_order>
+            <jump_host></jump_host>
+            <COMPONENTS>
+                <mac></mac>
+                <inband>false</inband>
+                <dhcp>True</dhcp>
+                <ip>10.32.11.194</ip>
+                <shortName>h3</shortName>
+                <port1></port1>
+                <link1></link1>
+                <interfaceName>eno2</interfaceName>
+                <routes>
+                    <route1>
+                        <network></network>
+                        <netmask></netmask>
+                        <gw></gw>
+                        <interface></interface>
+                    </route1>
+                </routes>
+                <sudo_required>false</sudo_required>
+            </COMPONENTS>
+        </Compute3>
+
+        <ManagmentServer>
+            <host>10.76.28.66</host>
+            <user>jenkins</user>
+            <password></password>
+            <type>HostDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS>
+                <mac></mac>
+                <inband>false</inband>
+                <dhcp>True</dhcp>
+                <ip>10.32.11.1</ip>
+                <shortName>mgmt</shortName>
+                <port1></port1>
+                <link1></link1>
+                <interfaceName>pairbond</interfaceName>
+                <routes>
+                    <route1>
+                        <network></network>
+                        <netmask></netmask>
+                        <gw></gw>
+                        <interface></interface>
+                    </route1>
+                </routes>
+                <sudo_required>false</sudo_required>
+
+            </COMPONENTS>
+        </ManagmentServer>
+
+        <NetworkBench>
+            <host>10.76.28.66</host>
+            <user>jenkins</user>
+            <password></password>
+            <type>NetworkDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </NetworkBench>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index ba8fba0..4143f93 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -36,6 +36,7 @@
         # (number of spine switch, number of leaf switch, dual-homed, description, minFlowCount - leaf (OvS), minFlowCount - leaf (BMv2))
         self.switchNames = {}
         self.switchNames[ '0x1' ] = [ "leaf1" ]
+        self.switchNames[ '0x2' ] = [ "leaf1", "leaf2" ]
         self.switchNames[ '2x2' ] = [ "leaf1", "leaf2", "spine101", "spine102" ]
         main.switchType = "ovs"
 
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index b5f61a7..5f9dd8a 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -96,6 +96,7 @@
             main.testSetUp.envSetupException( e )
 
         main.testSetUp.envSetupConclusion( stepResult )
+
     @staticmethod
     def getTopo():
         topo = dict()
@@ -222,11 +223,12 @@
     @staticmethod
     def loadChart( main ):
         try:
-            with open( "%s%s.chart" % ( main.configPath + main.forChart,
-                                        main.cfgName ) ) as chart:
+            filename = "%s%s.chart" % ( main.configPath + main.forChart,
+                                        main.cfgName )
+            with open( filename ) as chart:
                 main.pingChart = json.load( chart )
         except IOError:
-            main.log.warn( "No chart file found." )
+            main.log.warn( "No chart file found at %s" % filename )
 
     @staticmethod
     def loadHost( main ):
@@ -521,7 +523,7 @@
                                  onfail="Some bucket numbers are not as expected" )
 
     @staticmethod
-    def checkFlows( main, minFlowCount, tag="", dumpflows=True, sleep=10 ):
+    def checkFlows( main, minFlowCount, tag="", dumpFlows=True, sleep=10 ):
         main.step(
                 "Check whether the flow count is >= %s" % minFlowCount )
         if tag == "":
@@ -550,7 +552,7 @@
                 actual=flowCheck,
                 onpass="Flow status is correct!",
                 onfail="Flow status is wrong!" )
-        if dumpflows:
+        if dumpFlows:
             main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
                                         "flows",
                                         main.logdir,
@@ -645,7 +647,7 @@
         return
 
     @staticmethod
-    def pingAll( main, tag="", dumpflows=True, acceptableFailed=0, basedOnIp=False,
+    def pingAll( main, tag="", dumpFlows=True, acceptableFailed=0, basedOnIp=False,
                  sleep=10, retryAttempts=1, skipOnFail=False, useScapy=True ):
         '''
         Verify connectivity between hosts according to the ping chart
@@ -728,7 +730,72 @@
                 Testcaselib.cleanup( main, copyKarafLog=False )
                 main.skipCase()
 
-        if dumpflows:
+        if dumpFlows:
+            main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
+                                        "flows",
+                                        main.logdir,
+                                        tag + "_FlowsOn",
+                                        cliPort=main.Cluster.active(0).CLI.karafPort )
+            main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
+                                        "groups",
+                                        main.logdir,
+                                        tag + "_GroupsOn",
+                                        cliPort=main.Cluster.active(0).CLI.karafPort )
+
+    @staticmethod
+    def pingAllFabricIntfs( main, srcList, tag="", dumpFlows=True, skipOnFail=False ):
+        '''
+        Verify connectivity between hosts and their fabric interfaces
+        '''
+        main.log.report( "Check host connectivity with fabric" )
+        if tag == "":
+            tag = 'CASE%d' % main.CurrentTestCaseNumber
+        expect = main.TRUE
+        import json
+        import re
+        hostsJson = json.loads( main.Cluster.active( 0 ).hosts() )
+        netcfgJson = json.loads( main.Cluster.active( 0 ).getNetCfg( subjectClass='ports') )
+        for hostname in srcList:
+            try:
+                hostComponent = main.Network.hosts[ str( hostname ) ]
+                srcIface = hostComponent.interfaces[0].get( 'name' )
+                main.step( "Verify fabric connectivity for %s with tag %s" % ( str( hostname ), tag ) )
+                #Get host location, check netcfg for that port's ip
+                hostIp = hostComponent.getIPAddress( iface=srcIface )
+                main.log.warn( "Looking for %s" % hostIp )
+                ips = []
+                for obj in hostsJson:
+                    if hostIp in obj['ipAddresses']:
+                        for location in obj['locations']:
+                            main.log.debug( location )
+                            did = location['elementId'].encode( 'utf-8' )
+                            port = location['port'].encode( 'utf-8' )
+                            m = re.search( '\((\d+)\)', port )
+                            if m:
+                                port = m.group(1)
+                            portId = "%s/%s" % ( did, port )
+                            # Lookup ip assigned to this network port
+                            ips.extend( [ x.encode( 'utf-8' ) for x in netcfgJson[ portId ][ 'interfaces' ][0][ 'ips' ] ] )
+                ips = set( ips )
+                ipRE = r'(\d+\.\d+\.\d+\.\d+)/\d+|([\w,:]*)/\d+'
+                for ip in ips:
+                    ipMatch = re.search( ipRE, ip )
+                    if ipMatch:
+                        fabricIntfIp = ipMatch.group(1)
+                        main.log.debug( "Found %s as gateway ip for %s" % ( fabricIntfIp, hostname ) )
+                        pa = hostComponent.ping( fabricIntfIp, interface=srcIface )
+                        utilities.assert_equals( expect=expect, actual=pa,
+                                                 onpass="IP connectivity successfully tested",
+                                                 onfail="IP connectivity failed" )
+                        if pa != expect:
+                            Testcaselib.saveOnosDiagnostics( main )
+                            if skipOnFail:
+                                Testcaselib.cleanup( main, copyKarafLog=False )
+                                main.skipCase()
+            except ValueError:
+                main.log.exception( "Could not get gateway ip for %s" % hostname )
+
+        if dumpFlows:
             main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
                                         "flows",
                                         main.logdir,