Update tests for aether pods

- Update test for QA-POD
- SRStaging for testing connecting to Staging pod
- Add some functions for a kubernetes deployed cluster
- Connect to ONOS nodes with kubernetes
- Add option to connect to components through jump hosts
- Fixes for installing ONOS in custom locations
- Invoke python2 instead of python
- If using an ssh agent, also use that for pexpect ssh sessions,
  E.G. Jenkins initiated tests

Change-Id: I1fc345c8eab60a5b00c17e6ed677a63489a74a19
diff --git a/TestON/drivers/common/cli/emulator/scapyclidriver.py b/TestON/drivers/common/cli/emulator/scapyclidriver.py
index d340d5c..79a1956 100644
--- a/TestON/drivers/common/cli/emulator/scapyclidriver.py
+++ b/TestON/drivers/common/cli/emulator/scapyclidriver.py
@@ -40,12 +40,14 @@
         super( ScapyCliDriver, self ).__init__()
         self.handle = self
         self.name = None
-        self.home = None
+        self.home = "~/"
         self.wrapped = sys.modules[ __name__ ]
         self.flag = 0
         # TODO: Refactor driver to use these everywhere
         self.hostPrompt = "\$"
         self.scapyPrompt = ">>>"
+        self.sudoRequired = True
+        self.ifaceName = None
 
     def connect( self, **connectargs ):
         """
@@ -54,9 +56,17 @@
         try:
             for key in connectargs:
                 vars( self )[ key ] = connectargs[ key ]
-            self.home = self.options[ 'home' ] if 'home' in self.options.keys() else "~/"
-            self.name = self.options[ 'name' ]
-            self.ifaceName = self.options[ 'ifaceName' ] if 'ifaceName' in self.options.keys() else self.name + "-eth0"
+            for key in self.options:
+                if key == "home":
+                    self.home = self.options[ key ]
+                elif key == "name":
+                    self.name = self.options[ key ]
+                elif key == "sudo_required":
+                    self.sudoRequired = False if self.options[ key ] == "false" else True
+                elif key == "ifaceName":
+                    self.ifaceName = self.options[ key ]
+            if self.ifaceName is None:
+                self.ifaceName = self.name + "-eth0"
 
             # Parse route config
             self.routes = []
@@ -151,8 +161,21 @@
 
         try:
             main.log.debug( self.name + ": Starting scapy" )
-            self.handle.sendline( "sudo scapy" )
-            self.handle.expect( self.scapyPrompt )
+            if self.sudoRequired:
+                self.handle.sendline( "sudo scapy" )
+            else:
+                self.handle.sendline( "scapy" )
+            i = self.handle.expect( [ "not found", "password for", self.scapyPrompt ] )
+            if i == 1:
+                main.log.debug( "Sudo asking for password" )
+                main.log.sendline( self.pwd )
+                i = self.handle.expect( [ "not found", self.scapyPrompt ] )
+            if i == 0:
+                output = self.handle.before + self.handle.after
+                self.handle.expect( self.prompt )
+                output += self.handle.before + self.handle.after
+                main.log.debug( self.name + ": Scapy not installed, aborting test. \n" + output )
+                main.cleanAndExit()
             self.handle.sendline( "conf.color_theme = NoTheme()" )
             self.handle.expect( self.scapyPrompt )
             response = self.cleanOutput( self.handle.before )
@@ -1023,6 +1046,9 @@
         if gateway is None:
             main.log.error( self.name + ": Gateway is None, cannot set route" )
             return main.FALSE
+        if network is None or "None" in network:
+            main.log.error( self.name + ": Network is None, cannot set route" )
+            return main.FALSE
         try:
             cmdStr = 'conf.route.add( net="%s", gw="%s"' % ( network, gateway )
             if interface:
diff --git a/TestON/drivers/common/cli/hostdriver.py b/TestON/drivers/common/cli/hostdriver.py
index e14a38d..6175801 100644
--- a/TestON/drivers/common/cli/hostdriver.py
+++ b/TestON/drivers/common/cli/hostdriver.py
@@ -228,14 +228,12 @@
             self.handle.sendline( command )
             i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT ],
                                     timeout=wait + 5 )
+            response = self.handle.before
             if i == 1:
                 main.log.error(
                     self.name +
                     ": timeout when waiting for response" )
                 main.log.error( self.name + ": response: " + str( self.handle.before ) )
-            self.handle.sendline( "" )
-            self.handle.expect( self.prompt )
-            response = self.handle.before
             if re.search( ',\s0\%\spacket\sloss', response ):
                 main.log.info( self.name + ": no packets lost, host is reachable" )
                 return main.TRUE
diff --git a/TestON/drivers/common/cli/networkdriver.py b/TestON/drivers/common/cli/networkdriver.py
index 0f7b699..78f6464 100755
--- a/TestON/drivers/common/cli/networkdriver.py
+++ b/TestON/drivers/common/cli/networkdriver.py
@@ -509,7 +509,7 @@
             cannot reach each other"""
         hostComponentList = []
         for hostName in hostList:
-            hostComponent = self.hosts[ hostName ]
+            hostComponent = self.hosts[ str( hostName ) ]
             if hostComponent:
                 hostComponentList.append( hostComponent )
         try:
@@ -611,7 +611,7 @@
                 pingResponse += str( str( srcHost.shortName ) + " -> " )
                 for dstHost in dstComponentList:
                     failedPings = 0
-                    dstIP = dstHost.ip
+                    dstIP = dstHost.ip_address
                     assert dstIP, "Not able to get IP address of host {}".format( dstHost )
                     for iface in srcHost.interfaces:
                         # FIXME This only works if one iface name is configured
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index b0bf7cb..80dfe7c 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-
+# -*- coding: utf-8 -*-
 """
 OCT 13 2014
 Copyright 2014 Open Networking Foundation (ONF)
@@ -58,7 +58,9 @@
         self.handle = None
         self.karafUser = None
         self.karafPass = None
+        self.karafPort = None
         self.karafTimeout = None
+        self.address = None
 
         self.dockerPrompt = None
         self.graph = Graph()
@@ -71,7 +73,6 @@
         try:
             for key in connectargs:
                 vars( self )[ key ] = connectargs[ key ]
-            self.karafPrompt = self.user_name + "@root >"
             self.home = "~/onos"
             for key in self.options:
                 if key == "home":
@@ -84,12 +85,17 @@
                     self.dockerPrompt = self.options[ key ]
                 elif key == "karaf_timeout":
                     self.karafTimeout = self.options[ key ]
+                elif key == "karaf_port":
+                    self.karafPort = self.options[ key ]
             self.home = self.checkOptions( self.home, "~/onos" )
             self.karafUser = self.checkOptions( self.karafUser, self.user_name )
             self.karafPass = self.checkOptions( self.karafPass, self.pwd )
+            self.karafPort = self.checkOptions( self.karafPort, 8101 )
             self.dockerPrompt = self.checkOptions( self.dockerPrompt, "~/onos#" )
             self.karafTimeout = self.checkOptions( self.karafTimeout, 7200000  )
 
+            self.karafPrompt = self.karafUser + "@root >"
+
             for key in self.options:
                 if key == 'onosIp':
                     self.onosIp = self.options[ 'onosIp' ]
@@ -122,6 +128,7 @@
             self.handle.sendline( "cd " + self.home )
             self.handle.expect( self.prompt )
             if self.handle:
+                self.address = self.ip_address
                 return self.handle
             else:
                 main.log.info( "NO ONOS HANDLE" )
@@ -282,6 +289,7 @@
         and passed to startOnosCli from PARAMS file as str.
         """
         self.onosIp = ONOSIp
+        self.address = self.onosIp
         try:
             # Check if we are already in the cli
             self.handle.sendline( "" )
@@ -293,7 +301,7 @@
             # Not in CLI so login
             if self.inDocker:
                 # The Docker does not have all the wrapper scripts
-                startCliCommand = "ssh -p 8101 -o StrictHostKeyChecking=no %s@localhost" % self.karafUser
+                startCliCommand = "ssh -p %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s@localhost" % ( self.karafPort, self.karafUser )
             elif waitForStart:
                 # Wait for onos start ( onos-wait-for-start ) and enter onos cli
                 startCliCommand = "onos-wait-for-start " + str( ONOSIp )
@@ -342,6 +350,7 @@
             main.log.error( self.name + ":    " + self.handle.before )
             main.cleanAndExit()
         except Exception:
+            main.log.debug( self.handle.before + str( self.handle.after ) )
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
 
@@ -4931,6 +4940,28 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
 
+    def getAddress( self):
+        """
+        Get the onos ip address from the cli. This is usefull when connecting using
+        a container manager such as kubernetes. This function also sets self.address
+        the value from ONOS.
+
+        Returns:
+            The string value of the key or
+            None on Error
+        """
+        try:
+            output = self.summary()
+            address = json.loads( output ).get( 'node' )
+            self.address = address
+            return address
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
     def transactionalMapGet( self, keyName ):
         """
         CLI command to get the value of a key in a consistent map using
@@ -5317,6 +5348,39 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
 
+    def logList( self, saveValues=True ):
+        """
+        Gets the current log levels and optionally saves them
+        returns a dict of the log levels or
+        returns main.FALSE if Error occurred
+        """
+        try:
+            self.handle.sendline( "log:list" )
+            self.handle.expect( self.karafPrompt )
+
+            response = self.handle.before
+            logLevels = {}
+            for line in response.splitlines():
+                parsed = line.split('│')
+                logger = parsed[0].strip()
+                if len( parsed ) != 2 or 'Level' in parsed[1] or logger[0] == '─':
+                    continue
+                level = parsed[1].strip()
+                logLevels[ logger ] = level
+            if saveValues:
+                self.logLevels = logLevels
+            return logLevels
+        except pexpect.TIMEOUT:
+            main.log.exception( self.name + ": TIMEOUT exception found" )
+            main.cleanAndExit()
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
     def getGraphDict( self, timeout=60, includeHost=False ):
         """
         Return a dictionary which describes the latest network topology data as a
diff --git a/TestON/drivers/common/cli/onosclusterdriver.py b/TestON/drivers/common/cli/onosclusterdriver.py
index 11c44ae..d10b17f 100755
--- a/TestON/drivers/common/cli/onosclusterdriver.py
+++ b/TestON/drivers/common/cli/onosclusterdriver.py
@@ -68,7 +68,7 @@
         raise AttributeError( "Could not find the attribute %s in %r or it's component handles" % ( name, self ) )
 
     def __init__( self, name, ipAddress, CLI=None, REST=None, Bench=None, pos=None,
-                  userName=None, server=None, dockerPrompt=None ):
+                  userName=None, server=None, k8s=None, dockerPrompt=None ):
         # TODO: validate these arguments
         self.name = str( name )
         self.ipAddress = ipAddress
@@ -80,6 +80,7 @@
         self.ip_address = ipAddress
         self.user_name = userName
         self.server = server
+        self.k8s = k8s
         self.dockerPrompt = dockerPrompt
 
 class OnosClusterDriver( CLI ):
@@ -94,6 +95,7 @@
         self.useDocker = False
         self.dockerPrompt = None
         self.maxNodes = None
+        self.kubeConfig = None
         self.nodes = []
         super( OnosClusterDriver, self ).__init__()
 
@@ -129,6 +131,8 @@
                 elif key == "nodes":
                     # Maximum number of ONOS nodes to run, if there is any
                     self.maxNodes = self.options[ key ]
+                elif key == "kubeConfig":
+                    self.kubeConfig = self.options[ key ]
 
             self.home = self.checkOptions( self.home, "~/onos" )
             self.karafUser = self.checkOptions( self.karafUser, self.user_name )
@@ -139,43 +143,45 @@
             self.useDocker = self.checkOptions( self.useDocker, False )
             self.dockerPrompt = self.checkOptions( self.dockerPrompt, "~/onos#" )
             self.maxNodes = int( self.checkOptions( self.maxNodes, 100 ) )
+            self.kubeConfig = self.checkOptions( self.kubeConfig, None )
 
             self.name = self.options[ 'name' ]
 
-            # Grabs all OC environment variables based on max number of nodes
-            # TODO: Also support giving an ip range as a compononet option
-            self.onosIps = {}  # Dictionary of all possible ONOS ip
 
-            try:
-                if self.maxNodes:
-                    for i in range( self.maxNodes ):
-                        envString = "OC" + str( i + 1 )
-                        # If there is no more OC# then break the loop
-                        if os.getenv( envString ):
-                            self.onosIps[ envString ] = os.getenv( envString )
+            if not self.kubeConfig:
+                # Grabs all OC environment variables based on max number of nodes
+                # TODO: Also support giving an ip range as a compononet option
+                self.onosIps = {}  # Dictionary of all possible ONOS ip
+
+                try:
+                    if self.maxNodes:
+                        for i in range( self.maxNodes ):
+                            envString = "OC" + str( i + 1 )
+                            # If there is no more OC# then break the loop
+                            if os.getenv( envString ):
+                                self.onosIps[ envString ] = os.getenv( envString )
+                            else:
+                                self.maxNodes = len( self.onosIps )
+                                main.log.info( self.name +
+                                               ": Created cluster data with " +
+                                               str( self.maxNodes ) +
+                                               " maximum number" +
+                                               " of nodes" )
+                                break
+
+                        if not self.onosIps:
+                            main.log.info( "Could not read any environment variable"
+                                           + " please load a cell file with all" +
+                                            " onos IP" )
+                            self.maxNodes = None
                         else:
-                            self.maxNodes = len( self.onosIps )
-                            main.log.info( self.name +
-                                           ": Created cluster data with " +
-                                           str( self.maxNodes ) +
-                                           " maximum number" +
-                                           " of nodes" )
-                            break
-
-                    if not self.onosIps:
-                        main.log.info( "Could not read any environment variable"
-                                       + " please load a cell file with all" +
-                                        " onos IP" )
-                        self.maxNodes = None
-                    else:
-                        main.log.info( self.name + ": Found " +
-                                       str( self.onosIps.values() ) +
-                                       " ONOS IPs" )
-            except KeyError:
-                main.log.info( "Invalid environment variable" )
-            except Exception as inst:
-                main.log.error( "Uncaught exception: " + str( inst ) )
-
+                            main.log.info( self.name + ": Found " +
+                                           str( self.onosIps.values() ) +
+                                           " ONOS IPs" )
+                except KeyError:
+                    main.log.info( "Invalid environment variable" )
+                except Exception as inst:
+                    main.log.error( "Uncaught exception: " + str( inst ) )
             try:
                 if os.getenv( str( self.ip_address ) ) is not None:
                     self.ip_address = os.getenv( str( self.ip_address ) )
@@ -200,7 +206,41 @@
             if self.handle:
                 self.handle.sendline( "cd " + self.home )
                 self.handle.expect( "\$" )
+                if self.kubeConfig:
+                    # Try to get # of onos nodes using given kubernetes configuration
+                    names = self.kubectlGetPodNames( self.kubeConfig,
+                                                     main.params[ 'kubernetes' ][ 'namespace' ],
+                                                     main.params[ 'kubernetes' ][ 'appName' ] )
+                    self.podNames = names
+                    self.onosIps = {}  # Dictionary of all possible ONOS ip
+                    for i in range( 1, len( names ) + 1 ):
+                        self.onosIps[ 'OC%i' % i ] = self.ip_address
+                    self.maxNodes = len( names )
                 self.createComponents( prefix=prefix )
+                if self.kubeConfig:
+                    # Create Port Forwarding sessions for each controller
+                    for node in self.nodes:
+                        kubectl = node.k8s
+                        index = self.nodes.index( node )
+                        # Store each pod name in the k8s component
+                        kubectl.podName = self.podNames[ index ]
+                        # Setup port-forwarding and save the local port
+                        guiPort = 8181
+                        cliPort = 8101
+                        portsList = ""
+                        for port in [ guiPort, cliPort ]:
+                            localPort = port + index + 1
+                            portsList += "%s:%s " % ( localPort, port )
+                            if port == cliPort:
+                                node.CLI.karafPort = localPort
+                        main.log.info( "Setting up port forward for pod %s: [ %s ]" % ( self.podNames[ index ], portsList ) )
+                        pf = kubectl.kubectlPortForward( self.podNames[ index ],
+                                                         portsList,
+                                                         kubectl.kubeConfig,
+                                                         main.params[ 'kubernetes' ][ 'namespace' ] )
+                        if not pf:
+                            main.log.error( "Failed to create port forwarding" )
+                            return main.FALSE
                 return self.handle
             else:
                 main.log.info( "Failed to create ONOS handle" )
@@ -243,6 +283,8 @@
         clihost = main.componentDictionary[ name ][ 'COMPONENTS' ].get( "diff_clihost", "" )
         if clihost == "True":
             main.componentDictionary[ name ][ 'host' ] = host
+        home = main.componentDictionary[name]['COMPONENTS'].get( "onos_home", None )
+        main.componentDictionary[name]['home'] = self.checkOptions( home, None )
         main.componentDictionary[name]['type'] = "OnosCliDriver"
         main.componentDictionary[name]['connect_order'] = str( int( main.componentDictionary[name]['connect_order'] ) + 1 )
 
@@ -279,7 +321,6 @@
         Parse the cluster options to create an ONOS cli component with the given name
         """
         main.componentDictionary[name] = main.componentDictionary[self.name].copy()
-        main.log.debug( main.componentDictionary[name] )
         user = main.componentDictionary[name]['COMPONENTS'].get( "web_user", "onos" )
         main.componentDictionary[name]['user'] = self.checkOptions( user, "onos" )
         password = main.componentDictionary[name]['COMPONENTS'].get( "web_pass", "rocks" )
@@ -289,7 +330,6 @@
         main.componentDictionary[name]['port'] = self.checkOptions( port, "8181" )
         main.componentDictionary[name]['type'] = "OnosRestDriver"
         main.componentDictionary[name]['connect_order'] = str( int( main.componentDictionary[name]['connect_order'] ) + 1 )
-        main.log.debug( main.componentDictionary[name] )
 
     def createRestComponent( self, name, ipAddress ):
         """
@@ -328,7 +368,6 @@
         home = main.componentDictionary[name]['COMPONENTS'].get( "onos_home", None )
         main.componentDictionary[name]['home'] = self.checkOptions( home, None )
         main.componentDictionary[name]['connect_order'] = str( int( main.componentDictionary[name]['connect_order'] ) + 1 )
-        main.log.debug( main.componentDictionary[name] )
 
     def createBenchComponent( self, name ):
         """
@@ -372,10 +411,11 @@
         home = main.componentDictionary[name]['COMPONENTS'].get( "onos_home", None )
         main.componentDictionary[name]['home'] = self.checkOptions( home, None )
         # TODO: for now we use karaf user name and password also for logging to the onos nodes
-        main.componentDictionary[name]['user'] = self.karafUser
-        main.componentDictionary[name]['password'] = self.karafPass
+        # FIXME: We shouldn't use karaf* for this, what we want is another set of variables to
+        #        login to a shell on the server ONOS is running on
+        #main.componentDictionary[name]['user'] = self.karafUser
+        #main.componentDictionary[name]['password'] = self.karafPass
         main.componentDictionary[name]['connect_order'] = str( int( main.componentDictionary[name]['connect_order'] ) + 1 )
-        main.log.debug( main.componentDictionary[name] )
 
     def createServerComponent( self, name, ipAddress ):
         """
@@ -416,11 +456,14 @@
         restPrefix = prefix + "rest"
         benchPrefix = prefix + "bench"
         serverPrefix = prefix + "server"
+        k8sPrefix = prefix + "k8s"
         for i in xrange( 1, self.maxNodes + 1 ):
             cliName = cliPrefix + str( i  )
             restName = restPrefix + str( i )
             benchName = benchPrefix + str( i )
             serverName = serverPrefix + str( i )
+            if self.kubeConfig:
+                k8sName = k8sPrefix + str( i )
 
             # Unfortunately this means we need to have a cell set beofre running TestON,
             # Even if it is just the entire possible cluster size
@@ -430,6 +473,10 @@
             rest = self.createRestComponent( restName, ip )
             bench = self.createBenchComponent( benchName )
             server = self.createServerComponent( serverName, ip ) if createServer else None
+            k8s = self.createServerComponent( k8sName, ip ) if self.kubeConfig else None
+            if self.kubeConfig:
+                k8s.kubeConfig = self.kubeConfig
+                k8s.podName = None
             self.nodes.append( Controller( prefix + str( i ), ip, cli, rest, bench, i - 1,
-                                           self.user_name, server=server,
+                                           self.user_name, server=server, k8s=k8s,
                                            dockerPrompt=self.dockerPrompt ) )
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index e9cca52..78b8477 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -73,6 +73,10 @@
                     self.webUser = self.options[ key ]
                 elif key == "web_pass":
                     self.webPass = self.options[ key ]
+                elif key == "karaf_username":
+                    self.karafUser = self.options[ key ]
+                elif key == "karaf_password":
+                    self.karafPass = self.options[ key ]
 
             self.home = self.checkOptions( self.home, "~/onos" )
             self.maxNodes = self.checkOptions( self.maxNodes, 100 )
@@ -158,6 +162,7 @@
         response = main.TRUE
         try:
             if self.handle:
+                self.preDisconnect()
                 self.handle.sendline( "" )
                 self.handle.expect( self.prompt )
                 self.handle.sendline( "exit" )
@@ -392,7 +397,7 @@
                 elif i == 4:
                     # Prompt returned
                     break
-            main.log.debug( output )
+            main.log.debug( self.name + ": " + output )
             return ret
         except pexpect.TIMEOUT:
             main.log.exception( self.name + ": TIMEOUT exception found" )
@@ -446,7 +451,7 @@
                 elif i == 5:
                     # Prompt returned
                     break
-            main.log.debug( output )
+            main.log.debug( self.name + ": " + output )
             return ret
         except pexpect.TIMEOUT:
             main.log.exception( self.name + ": TIMEOUT exception found" )
@@ -812,7 +817,6 @@
             ~/<self.home>/tools/test/cells/
         """
         try:
-
             # Variable initialization
             cellDirectory = self.home + "/tools/test/cells/"
             # We want to create the cell file in the dependencies directory
@@ -1008,7 +1012,7 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
 
-    def onosCli( self, ONOSIp, cmdstr, timeout=60 ):
+    def onosCli( self, ONOSIp, cmdstr, cliPort=8101, waitForStart=False, timeout=60 ):
         """
         Uses 'onos' command to send various ONOS CLI arguments.
         Required:
@@ -1038,17 +1042,17 @@
             self.handle.sendline( "" )
             self.handle.expect( self.prompt )
 
-            self.handle.sendline( "onos-wait-for-start " + ONOSIp )
-            i = self.handle.expect( [ self.prompt, "Password: " ] )
-            if i == 1:
-                self.handle.sendline( self.pwd )
-                self.handle.expect( self.prompt )
-
-            self.handle.sendline( "ssh -q -p 8101 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s %s " % ( ONOSIp, cmdstr ) )
+            if waitForStart:
+                self.handle.sendline( "onos-wait-for-start " + ONOSIp )
+                i = self.handle.expect( [ self.prompt, "Password: " ] )
+                if i == 1:
+                    self.handle.sendline( self.pwd )
+                    self.handle.expect( self.prompt )
+            self.handle.sendline( "ssh -q -p %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s@%s %s " % ( cliPort, self.karafUser, ONOSIp, cmdstr ) )
             i = self.handle.expect( [ self.prompt, "Password: ", pexpect.TIMEOUT ], timeout=timeout )
             if i == 1:
-                self.handle.sendline( self.pwd )
-                i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT ], timeout=timeout )
+                self.handle.sendline( self.karafPass )
+                i = self.handle.expect( [ self.prompt, "Password:", pexpect.TIMEOUT ], timeout=timeout )
             if i == 0:
                 handleBefore = self.handle.before
                 main.log.info( self.name + ": Command sent successfully" )
@@ -1058,12 +1062,18 @@
                 returnString = handleBefore
                 return returnString
             elif i == 1:
+                main.log.error( self.name + ": Incorrect password for ONOS cli" )
+                self.handle.send( "\x03" )  # Control-C
+                self.handle.expect( self.prompt )
+                return main.FALSE
+            elif i == 2:
                 main.log.error( self.name + ": Timeout when sending " + cmdstr )
-                main.log.debug( self.handle.before )
+                main.log.debug( self.name + ": " + self.handle.before )
                 self.handle.send( "\x03" )  # Control-C
                 self.handle.expect( self.prompt )
                 return main.FALSE
         except pexpect.TIMEOUT:
+            main.log.debug( self.handle.before + str( self.handle.after ) )
             main.log.exception( self.name + ": Timeout when sending " + cmdstr )
             return main.FALSE
         except pexpect.EOF:
@@ -1107,7 +1117,7 @@
             elif i == 2:
                 # timeout
                 main.log.error( self.name + ": Failed to secure ssh on " + node )
-                main.log.debug( self.handle.before )
+                main.log.debug( self.name + ": " + self.handle.before )
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
             main.log.error( self.name + ":    " + self.handle.before )
@@ -1755,7 +1765,7 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
 
-    def dumpONOSCmd( self, ONOSIp, CMD, destDir, filename, options="", timeout=60 ):
+    def dumpONOSCmd( self, ONOSIp, CMD, destDir, filename, options="", cliPort=8101, timeout=60 ):
         """
         Dump Cmd to a desired directory.
         For debugging purposes, you may want to use
@@ -1780,7 +1790,7 @@
         if destDir[ -1: ] != "/":
             destDir += "/"
         cmd = CMD + " " + options + " > " + str( destDir ) + str( filename ) + localtime
-        return self.onosCli( ONOSIp, cmd, timeout=timeout )
+        return self.onosCli( ONOSIp, cmd, cliPort=cliPort, timeout=timeout )
 
     def cpLogsToDir( self, logToCopy,
                      destDir, copyFileName="" ):