Look at port stats to choose port

- In TOST port failure tests, look at port stats to
  determine which link is carrying the most traffic
  so we can bring it down
- Add eNB leaf-spine link down case

Change-Id: Ia13f3d41e836deaf21dd93574a39ccd954dd488f
diff --git a/TestON/drivers/common/api/controller/onosrestdriver.py b/TestON/drivers/common/api/controller/onosrestdriver.py
index 7f51bcd..403b436 100755
--- a/TestON/drivers/common/api/controller/onosrestdriver.py
+++ b/TestON/drivers/common/api/controller/onosrestdriver.py
@@ -2345,3 +2345,41 @@
         except Exception:
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
+
+    def portstats( self, ip="DEFAULT", port="DEFAULT" ):
+        """
+        Description:
+            Gets the portstats for each port in ONOS
+        Returns:
+            A list of dicts containing device id and a list of dicts containing the
+            port statistics for each port.
+            Returns main.FALSE if error on request;
+            Returns None for exception
+        """
+        try:
+            output = None
+            if ip == "DEFAULT":
+                main.log.warn( self.name + ": No ip given, reverting to ip from topo file" )
+                ip = self.ip_address
+            if port == "DEFAULT":
+                main.log.warn( self.name + ": No port given, reverting to port " +
+                               "from topo file" )
+                port = self.port
+            response = self.send( url="/statistics/ports", ip = ip, port = port )
+            if response:
+                if 200 <= response[ 0 ] <= 299:
+                    output = response[ 1 ]
+                    a = json.loads( output ).get( 'statistics' )
+                    assert a is not None, "Error parsing json object"
+                    b = json.dumps( a )
+                    return b
+                else:
+                    main.log.error( "Error with REST request, response was: %s: %s" %
+                                    ( response[ 0 ], response[ 1 ] ) )
+                    return main.FALSE
+        except ( AttributeError, AssertionError, TypeError ):
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
diff --git a/TestON/drivers/common/cli/emulator/scapyclidriver.py b/TestON/drivers/common/cli/emulator/scapyclidriver.py
index 79a1956..c4708bc 100644
--- a/TestON/drivers/common/cli/emulator/scapyclidriver.py
+++ b/TestON/drivers/common/cli/emulator/scapyclidriver.py
@@ -792,7 +792,7 @@
             self.handle.expect( self.scapyPrompt )
             response = self.cleanOutput( self.handle.before )
             main.log.debug( self.name + ": Send packet response: {}".format( response ) )
-            if "Traceback" in response:
+            if "Traceback" in response or "Errno" in response or "Error" in response:
                 # KeyError, SyntaxError, ...
                 main.log.error( self.name + ": Error in sending command: " + response )
                 return main.FALSE
@@ -915,6 +915,10 @@
             else:
                 self.handle.sendline( "pkts.summary()")
             output = self.clearBuffer()
+            if "Traceback" in output or "Errno" in output or "Error" in output:
+                # KeyError, SyntaxError, IOError, NameError, ...
+                main.log.error( self.name + ": Error in sending command: " + output )
+                main.cleanAndExit()
         except pexpect.TIMEOUT:
             main.log.exception( self.name + ": Command timed out" )
             return None
@@ -1081,7 +1085,7 @@
             gw = route.get( 'gw' )
             iface = route.get( 'interface' )
             returnValues .append( self.addRoute( "%s/%s" % ( route.get( 'network' ), route.get( 'netmask' ) ),
-                                                 gw if gw else main.Cluster.active(0).ipAddress,
+                                                 gw if gw else main.Cluster.active(0).address,
                                                  interface=iface if iface else self.interfaces[ 0 ].get( 'name' ) ) )
         return returnValues
 
diff --git a/TestON/drivers/common/cli/onosclusterdriver.py b/TestON/drivers/common/cli/onosclusterdriver.py
index d10b17f..b0f924b 100755
--- a/TestON/drivers/common/cli/onosclusterdriver.py
+++ b/TestON/drivers/common/cli/onosclusterdriver.py
@@ -233,6 +233,8 @@
                             portsList += "%s:%s " % ( localPort, port )
                             if port == cliPort:
                                 node.CLI.karafPort = localPort
+                            elif port == guiPort:
+                                node.REST.port = localPort
                         main.log.info( "Setting up port forward for pod %s: [ %s ]" % ( self.podNames[ index ], portsList ) )
                         pf = kubectl.kubectlPortForward( self.podNames[ index ],
                                                          portsList,
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index aa1e9f9..e2265ce 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -1065,7 +1065,7 @@
         """
 
         try:
-            cmdStr = "kubectl %s %s get pods -o=custom-columns=NAME:.metadata.name,NODE:.spec.nodeName %s " % (
+            cmdStr = "kubectl %s %s get pods -o wide %s " % (
                         "--kubeconfig %s" % kubeconfig if kubeconfig else "",
                         "-n %s" % namespace if namespace else "",
                         " > %s" % dstPath if dstPath else "" )