Update tests for aether pods
- Update test for QA-POD
- SRStaging for testing connecting to Staging pod
- Add some functions for a kubernetes deployed cluster
- Connect to ONOS nodes with kubernetes
- Add option to connect to components through jump hosts
- Fixes for installing ONOS in custom locations
- Invoke python2 instead of python
- If using an ssh agent, also use that for pexpect ssh sessions,
E.G. Jenkins initiated tests
Change-Id: I1fc345c8eab60a5b00c17e6ed677a63489a74a19
diff --git a/TestON/drivers/common/cli/emulator/scapyclidriver.py b/TestON/drivers/common/cli/emulator/scapyclidriver.py
index d340d5c..79a1956 100644
--- a/TestON/drivers/common/cli/emulator/scapyclidriver.py
+++ b/TestON/drivers/common/cli/emulator/scapyclidriver.py
@@ -40,12 +40,14 @@
super( ScapyCliDriver, self ).__init__()
self.handle = self
self.name = None
- self.home = None
+ self.home = "~/"
self.wrapped = sys.modules[ __name__ ]
self.flag = 0
# TODO: Refactor driver to use these everywhere
self.hostPrompt = "\$"
self.scapyPrompt = ">>>"
+ self.sudoRequired = True
+ self.ifaceName = None
def connect( self, **connectargs ):
"""
@@ -54,9 +56,17 @@
try:
for key in connectargs:
vars( self )[ key ] = connectargs[ key ]
- self.home = self.options[ 'home' ] if 'home' in self.options.keys() else "~/"
- self.name = self.options[ 'name' ]
- self.ifaceName = self.options[ 'ifaceName' ] if 'ifaceName' in self.options.keys() else self.name + "-eth0"
+ for key in self.options:
+ if key == "home":
+ self.home = self.options[ key ]
+ elif key == "name":
+ self.name = self.options[ key ]
+ elif key == "sudo_required":
+ self.sudoRequired = False if self.options[ key ] == "false" else True
+ elif key == "ifaceName":
+ self.ifaceName = self.options[ key ]
+ if self.ifaceName is None:
+ self.ifaceName = self.name + "-eth0"
# Parse route config
self.routes = []
@@ -151,8 +161,21 @@
try:
main.log.debug( self.name + ": Starting scapy" )
- self.handle.sendline( "sudo scapy" )
- self.handle.expect( self.scapyPrompt )
+ if self.sudoRequired:
+ self.handle.sendline( "sudo scapy" )
+ else:
+ self.handle.sendline( "scapy" )
+ i = self.handle.expect( [ "not found", "password for", self.scapyPrompt ] )
+ if i == 1:
+ main.log.debug( "Sudo asking for password" )
+ main.log.sendline( self.pwd )
+ i = self.handle.expect( [ "not found", self.scapyPrompt ] )
+ if i == 0:
+ output = self.handle.before + self.handle.after
+ self.handle.expect( self.prompt )
+ output += self.handle.before + self.handle.after
+ main.log.debug( self.name + ": Scapy not installed, aborting test. \n" + output )
+ main.cleanAndExit()
self.handle.sendline( "conf.color_theme = NoTheme()" )
self.handle.expect( self.scapyPrompt )
response = self.cleanOutput( self.handle.before )
@@ -1023,6 +1046,9 @@
if gateway is None:
main.log.error( self.name + ": Gateway is None, cannot set route" )
return main.FALSE
+ if network is None or "None" in network:
+ main.log.error( self.name + ": Network is None, cannot set route" )
+ return main.FALSE
try:
cmdStr = 'conf.route.add( net="%s", gw="%s"' % ( network, gateway )
if interface:
diff --git a/TestON/drivers/common/cli/hostdriver.py b/TestON/drivers/common/cli/hostdriver.py
index e14a38d..6175801 100644
--- a/TestON/drivers/common/cli/hostdriver.py
+++ b/TestON/drivers/common/cli/hostdriver.py
@@ -228,14 +228,12 @@
self.handle.sendline( command )
i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT ],
timeout=wait + 5 )
+ response = self.handle.before
if i == 1:
main.log.error(
self.name +
": timeout when waiting for response" )
main.log.error( self.name + ": response: " + str( self.handle.before ) )
- self.handle.sendline( "" )
- self.handle.expect( self.prompt )
- response = self.handle.before
if re.search( ',\s0\%\spacket\sloss', response ):
main.log.info( self.name + ": no packets lost, host is reachable" )
return main.TRUE
diff --git a/TestON/drivers/common/cli/networkdriver.py b/TestON/drivers/common/cli/networkdriver.py
index 0f7b699..78f6464 100755
--- a/TestON/drivers/common/cli/networkdriver.py
+++ b/TestON/drivers/common/cli/networkdriver.py
@@ -509,7 +509,7 @@
cannot reach each other"""
hostComponentList = []
for hostName in hostList:
- hostComponent = self.hosts[ hostName ]
+ hostComponent = self.hosts[ str( hostName ) ]
if hostComponent:
hostComponentList.append( hostComponent )
try:
@@ -611,7 +611,7 @@
pingResponse += str( str( srcHost.shortName ) + " -> " )
for dstHost in dstComponentList:
failedPings = 0
- dstIP = dstHost.ip
+ dstIP = dstHost.ip_address
assert dstIP, "Not able to get IP address of host {}".format( dstHost )
for iface in srcHost.interfaces:
# FIXME This only works if one iface name is configured
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index b0bf7cb..80dfe7c 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-
+# -*- coding: utf-8 -*-
"""
OCT 13 2014
Copyright 2014 Open Networking Foundation (ONF)
@@ -58,7 +58,9 @@
self.handle = None
self.karafUser = None
self.karafPass = None
+ self.karafPort = None
self.karafTimeout = None
+ self.address = None
self.dockerPrompt = None
self.graph = Graph()
@@ -71,7 +73,6 @@
try:
for key in connectargs:
vars( self )[ key ] = connectargs[ key ]
- self.karafPrompt = self.user_name + "@root >"
self.home = "~/onos"
for key in self.options:
if key == "home":
@@ -84,12 +85,17 @@
self.dockerPrompt = self.options[ key ]
elif key == "karaf_timeout":
self.karafTimeout = self.options[ key ]
+ elif key == "karaf_port":
+ self.karafPort = self.options[ key ]
self.home = self.checkOptions( self.home, "~/onos" )
self.karafUser = self.checkOptions( self.karafUser, self.user_name )
self.karafPass = self.checkOptions( self.karafPass, self.pwd )
+ self.karafPort = self.checkOptions( self.karafPort, 8101 )
self.dockerPrompt = self.checkOptions( self.dockerPrompt, "~/onos#" )
self.karafTimeout = self.checkOptions( self.karafTimeout, 7200000 )
+ self.karafPrompt = self.karafUser + "@root >"
+
for key in self.options:
if key == 'onosIp':
self.onosIp = self.options[ 'onosIp' ]
@@ -122,6 +128,7 @@
self.handle.sendline( "cd " + self.home )
self.handle.expect( self.prompt )
if self.handle:
+ self.address = self.ip_address
return self.handle
else:
main.log.info( "NO ONOS HANDLE" )
@@ -282,6 +289,7 @@
and passed to startOnosCli from PARAMS file as str.
"""
self.onosIp = ONOSIp
+ self.address = self.onosIp
try:
# Check if we are already in the cli
self.handle.sendline( "" )
@@ -293,7 +301,7 @@
# Not in CLI so login
if self.inDocker:
# The Docker does not have all the wrapper scripts
- startCliCommand = "ssh -p 8101 -o StrictHostKeyChecking=no %s@localhost" % self.karafUser
+ startCliCommand = "ssh -p %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s@localhost" % ( self.karafPort, self.karafUser )
elif waitForStart:
# Wait for onos start ( onos-wait-for-start ) and enter onos cli
startCliCommand = "onos-wait-for-start " + str( ONOSIp )
@@ -342,6 +350,7 @@
main.log.error( self.name + ": " + self.handle.before )
main.cleanAndExit()
except Exception:
+ main.log.debug( self.handle.before + str( self.handle.after ) )
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
@@ -4931,6 +4940,28 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+ def getAddress( self):
+ """
+ Get the onos ip address from the cli. This is usefull when connecting using
+ a container manager such as kubernetes. This function also sets self.address
+ the value from ONOS.
+
+ Returns:
+ The string value of the key or
+ None on Error
+ """
+ try:
+ output = self.summary()
+ address = json.loads( output ).get( 'node' )
+ self.address = address
+ return address
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
def transactionalMapGet( self, keyName ):
"""
CLI command to get the value of a key in a consistent map using
@@ -5317,6 +5348,39 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+ def logList( self, saveValues=True ):
+ """
+ Gets the current log levels and optionally saves them
+ returns a dict of the log levels or
+ returns main.FALSE if Error occurred
+ """
+ try:
+ self.handle.sendline( "log:list" )
+ self.handle.expect( self.karafPrompt )
+
+ response = self.handle.before
+ logLevels = {}
+ for line in response.splitlines():
+ parsed = line.split('│')
+ logger = parsed[0].strip()
+ if len( parsed ) != 2 or 'Level' in parsed[1] or logger[0] == '─':
+ continue
+ level = parsed[1].strip()
+ logLevels[ logger ] = level
+ if saveValues:
+ self.logLevels = logLevels
+ return logLevels
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.cleanAndExit()
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
def getGraphDict( self, timeout=60, includeHost=False ):
"""
Return a dictionary which describes the latest network topology data as a
diff --git a/TestON/drivers/common/cli/onosclusterdriver.py b/TestON/drivers/common/cli/onosclusterdriver.py
index 11c44ae..d10b17f 100755
--- a/TestON/drivers/common/cli/onosclusterdriver.py
+++ b/TestON/drivers/common/cli/onosclusterdriver.py
@@ -68,7 +68,7 @@
raise AttributeError( "Could not find the attribute %s in %r or it's component handles" % ( name, self ) )
def __init__( self, name, ipAddress, CLI=None, REST=None, Bench=None, pos=None,
- userName=None, server=None, dockerPrompt=None ):
+ userName=None, server=None, k8s=None, dockerPrompt=None ):
# TODO: validate these arguments
self.name = str( name )
self.ipAddress = ipAddress
@@ -80,6 +80,7 @@
self.ip_address = ipAddress
self.user_name = userName
self.server = server
+ self.k8s = k8s
self.dockerPrompt = dockerPrompt
class OnosClusterDriver( CLI ):
@@ -94,6 +95,7 @@
self.useDocker = False
self.dockerPrompt = None
self.maxNodes = None
+ self.kubeConfig = None
self.nodes = []
super( OnosClusterDriver, self ).__init__()
@@ -129,6 +131,8 @@
elif key == "nodes":
# Maximum number of ONOS nodes to run, if there is any
self.maxNodes = self.options[ key ]
+ elif key == "kubeConfig":
+ self.kubeConfig = self.options[ key ]
self.home = self.checkOptions( self.home, "~/onos" )
self.karafUser = self.checkOptions( self.karafUser, self.user_name )
@@ -139,43 +143,45 @@
self.useDocker = self.checkOptions( self.useDocker, False )
self.dockerPrompt = self.checkOptions( self.dockerPrompt, "~/onos#" )
self.maxNodes = int( self.checkOptions( self.maxNodes, 100 ) )
+ self.kubeConfig = self.checkOptions( self.kubeConfig, None )
self.name = self.options[ 'name' ]
- # Grabs all OC environment variables based on max number of nodes
- # TODO: Also support giving an ip range as a compononet option
- self.onosIps = {} # Dictionary of all possible ONOS ip
- try:
- if self.maxNodes:
- for i in range( self.maxNodes ):
- envString = "OC" + str( i + 1 )
- # If there is no more OC# then break the loop
- if os.getenv( envString ):
- self.onosIps[ envString ] = os.getenv( envString )
+ if not self.kubeConfig:
+ # Grabs all OC environment variables based on max number of nodes
+ # TODO: Also support giving an ip range as a compononet option
+ self.onosIps = {} # Dictionary of all possible ONOS ip
+
+ try:
+ if self.maxNodes:
+ for i in range( self.maxNodes ):
+ envString = "OC" + str( i + 1 )
+ # If there is no more OC# then break the loop
+ if os.getenv( envString ):
+ self.onosIps[ envString ] = os.getenv( envString )
+ else:
+ self.maxNodes = len( self.onosIps )
+ main.log.info( self.name +
+ ": Created cluster data with " +
+ str( self.maxNodes ) +
+ " maximum number" +
+ " of nodes" )
+ break
+
+ if not self.onosIps:
+ main.log.info( "Could not read any environment variable"
+ + " please load a cell file with all" +
+ " onos IP" )
+ self.maxNodes = None
else:
- self.maxNodes = len( self.onosIps )
- main.log.info( self.name +
- ": Created cluster data with " +
- str( self.maxNodes ) +
- " maximum number" +
- " of nodes" )
- break
-
- if not self.onosIps:
- main.log.info( "Could not read any environment variable"
- + " please load a cell file with all" +
- " onos IP" )
- self.maxNodes = None
- else:
- main.log.info( self.name + ": Found " +
- str( self.onosIps.values() ) +
- " ONOS IPs" )
- except KeyError:
- main.log.info( "Invalid environment variable" )
- except Exception as inst:
- main.log.error( "Uncaught exception: " + str( inst ) )
-
+ main.log.info( self.name + ": Found " +
+ str( self.onosIps.values() ) +
+ " ONOS IPs" )
+ except KeyError:
+ main.log.info( "Invalid environment variable" )
+ except Exception as inst:
+ main.log.error( "Uncaught exception: " + str( inst ) )
try:
if os.getenv( str( self.ip_address ) ) is not None:
self.ip_address = os.getenv( str( self.ip_address ) )
@@ -200,7 +206,41 @@
if self.handle:
self.handle.sendline( "cd " + self.home )
self.handle.expect( "\$" )
+ if self.kubeConfig:
+ # Try to get # of onos nodes using given kubernetes configuration
+ names = self.kubectlGetPodNames( self.kubeConfig,
+ main.params[ 'kubernetes' ][ 'namespace' ],
+ main.params[ 'kubernetes' ][ 'appName' ] )
+ self.podNames = names
+ self.onosIps = {} # Dictionary of all possible ONOS ip
+ for i in range( 1, len( names ) + 1 ):
+ self.onosIps[ 'OC%i' % i ] = self.ip_address
+ self.maxNodes = len( names )
self.createComponents( prefix=prefix )
+ if self.kubeConfig:
+ # Create Port Forwarding sessions for each controller
+ for node in self.nodes:
+ kubectl = node.k8s
+ index = self.nodes.index( node )
+ # Store each pod name in the k8s component
+ kubectl.podName = self.podNames[ index ]
+ # Setup port-forwarding and save the local port
+ guiPort = 8181
+ cliPort = 8101
+ portsList = ""
+ for port in [ guiPort, cliPort ]:
+ localPort = port + index + 1
+ portsList += "%s:%s " % ( localPort, port )
+ if port == cliPort:
+ node.CLI.karafPort = localPort
+ main.log.info( "Setting up port forward for pod %s: [ %s ]" % ( self.podNames[ index ], portsList ) )
+ pf = kubectl.kubectlPortForward( self.podNames[ index ],
+ portsList,
+ kubectl.kubeConfig,
+ main.params[ 'kubernetes' ][ 'namespace' ] )
+ if not pf:
+ main.log.error( "Failed to create port forwarding" )
+ return main.FALSE
return self.handle
else:
main.log.info( "Failed to create ONOS handle" )
@@ -243,6 +283,8 @@
clihost = main.componentDictionary[ name ][ 'COMPONENTS' ].get( "diff_clihost", "" )
if clihost == "True":
main.componentDictionary[ name ][ 'host' ] = host
+ home = main.componentDictionary[name]['COMPONENTS'].get( "onos_home", None )
+ main.componentDictionary[name]['home'] = self.checkOptions( home, None )
main.componentDictionary[name]['type'] = "OnosCliDriver"
main.componentDictionary[name]['connect_order'] = str( int( main.componentDictionary[name]['connect_order'] ) + 1 )
@@ -279,7 +321,6 @@
Parse the cluster options to create an ONOS cli component with the given name
"""
main.componentDictionary[name] = main.componentDictionary[self.name].copy()
- main.log.debug( main.componentDictionary[name] )
user = main.componentDictionary[name]['COMPONENTS'].get( "web_user", "onos" )
main.componentDictionary[name]['user'] = self.checkOptions( user, "onos" )
password = main.componentDictionary[name]['COMPONENTS'].get( "web_pass", "rocks" )
@@ -289,7 +330,6 @@
main.componentDictionary[name]['port'] = self.checkOptions( port, "8181" )
main.componentDictionary[name]['type'] = "OnosRestDriver"
main.componentDictionary[name]['connect_order'] = str( int( main.componentDictionary[name]['connect_order'] ) + 1 )
- main.log.debug( main.componentDictionary[name] )
def createRestComponent( self, name, ipAddress ):
"""
@@ -328,7 +368,6 @@
home = main.componentDictionary[name]['COMPONENTS'].get( "onos_home", None )
main.componentDictionary[name]['home'] = self.checkOptions( home, None )
main.componentDictionary[name]['connect_order'] = str( int( main.componentDictionary[name]['connect_order'] ) + 1 )
- main.log.debug( main.componentDictionary[name] )
def createBenchComponent( self, name ):
"""
@@ -372,10 +411,11 @@
home = main.componentDictionary[name]['COMPONENTS'].get( "onos_home", None )
main.componentDictionary[name]['home'] = self.checkOptions( home, None )
# TODO: for now we use karaf user name and password also for logging to the onos nodes
- main.componentDictionary[name]['user'] = self.karafUser
- main.componentDictionary[name]['password'] = self.karafPass
+ # FIXME: We shouldn't use karaf* for this, what we want is another set of variables to
+ # login to a shell on the server ONOS is running on
+ #main.componentDictionary[name]['user'] = self.karafUser
+ #main.componentDictionary[name]['password'] = self.karafPass
main.componentDictionary[name]['connect_order'] = str( int( main.componentDictionary[name]['connect_order'] ) + 1 )
- main.log.debug( main.componentDictionary[name] )
def createServerComponent( self, name, ipAddress ):
"""
@@ -416,11 +456,14 @@
restPrefix = prefix + "rest"
benchPrefix = prefix + "bench"
serverPrefix = prefix + "server"
+ k8sPrefix = prefix + "k8s"
for i in xrange( 1, self.maxNodes + 1 ):
cliName = cliPrefix + str( i )
restName = restPrefix + str( i )
benchName = benchPrefix + str( i )
serverName = serverPrefix + str( i )
+ if self.kubeConfig:
+ k8sName = k8sPrefix + str( i )
# Unfortunately this means we need to have a cell set beofre running TestON,
# Even if it is just the entire possible cluster size
@@ -430,6 +473,10 @@
rest = self.createRestComponent( restName, ip )
bench = self.createBenchComponent( benchName )
server = self.createServerComponent( serverName, ip ) if createServer else None
+ k8s = self.createServerComponent( k8sName, ip ) if self.kubeConfig else None
+ if self.kubeConfig:
+ k8s.kubeConfig = self.kubeConfig
+ k8s.podName = None
self.nodes.append( Controller( prefix + str( i ), ip, cli, rest, bench, i - 1,
- self.user_name, server=server,
+ self.user_name, server=server, k8s=k8s,
dockerPrompt=self.dockerPrompt ) )
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index e9cca52..78b8477 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -73,6 +73,10 @@
self.webUser = self.options[ key ]
elif key == "web_pass":
self.webPass = self.options[ key ]
+ elif key == "karaf_username":
+ self.karafUser = self.options[ key ]
+ elif key == "karaf_password":
+ self.karafPass = self.options[ key ]
self.home = self.checkOptions( self.home, "~/onos" )
self.maxNodes = self.checkOptions( self.maxNodes, 100 )
@@ -158,6 +162,7 @@
response = main.TRUE
try:
if self.handle:
+ self.preDisconnect()
self.handle.sendline( "" )
self.handle.expect( self.prompt )
self.handle.sendline( "exit" )
@@ -392,7 +397,7 @@
elif i == 4:
# Prompt returned
break
- main.log.debug( output )
+ main.log.debug( self.name + ": " + output )
return ret
except pexpect.TIMEOUT:
main.log.exception( self.name + ": TIMEOUT exception found" )
@@ -446,7 +451,7 @@
elif i == 5:
# Prompt returned
break
- main.log.debug( output )
+ main.log.debug( self.name + ": " + output )
return ret
except pexpect.TIMEOUT:
main.log.exception( self.name + ": TIMEOUT exception found" )
@@ -812,7 +817,6 @@
~/<self.home>/tools/test/cells/
"""
try:
-
# Variable initialization
cellDirectory = self.home + "/tools/test/cells/"
# We want to create the cell file in the dependencies directory
@@ -1008,7 +1012,7 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
- def onosCli( self, ONOSIp, cmdstr, timeout=60 ):
+ def onosCli( self, ONOSIp, cmdstr, cliPort=8101, waitForStart=False, timeout=60 ):
"""
Uses 'onos' command to send various ONOS CLI arguments.
Required:
@@ -1038,17 +1042,17 @@
self.handle.sendline( "" )
self.handle.expect( self.prompt )
- self.handle.sendline( "onos-wait-for-start " + ONOSIp )
- i = self.handle.expect( [ self.prompt, "Password: " ] )
- if i == 1:
- self.handle.sendline( self.pwd )
- self.handle.expect( self.prompt )
-
- self.handle.sendline( "ssh -q -p 8101 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s %s " % ( ONOSIp, cmdstr ) )
+ if waitForStart:
+ self.handle.sendline( "onos-wait-for-start " + ONOSIp )
+ i = self.handle.expect( [ self.prompt, "Password: " ] )
+ if i == 1:
+ self.handle.sendline( self.pwd )
+ self.handle.expect( self.prompt )
+ self.handle.sendline( "ssh -q -p %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s@%s %s " % ( cliPort, self.karafUser, ONOSIp, cmdstr ) )
i = self.handle.expect( [ self.prompt, "Password: ", pexpect.TIMEOUT ], timeout=timeout )
if i == 1:
- self.handle.sendline( self.pwd )
- i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT ], timeout=timeout )
+ self.handle.sendline( self.karafPass )
+ i = self.handle.expect( [ self.prompt, "Password:", pexpect.TIMEOUT ], timeout=timeout )
if i == 0:
handleBefore = self.handle.before
main.log.info( self.name + ": Command sent successfully" )
@@ -1058,12 +1062,18 @@
returnString = handleBefore
return returnString
elif i == 1:
+ main.log.error( self.name + ": Incorrect password for ONOS cli" )
+ self.handle.send( "\x03" ) # Control-C
+ self.handle.expect( self.prompt )
+ return main.FALSE
+ elif i == 2:
main.log.error( self.name + ": Timeout when sending " + cmdstr )
- main.log.debug( self.handle.before )
+ main.log.debug( self.name + ": " + self.handle.before )
self.handle.send( "\x03" ) # Control-C
self.handle.expect( self.prompt )
return main.FALSE
except pexpect.TIMEOUT:
+ main.log.debug( self.handle.before + str( self.handle.after ) )
main.log.exception( self.name + ": Timeout when sending " + cmdstr )
return main.FALSE
except pexpect.EOF:
@@ -1107,7 +1117,7 @@
elif i == 2:
# timeout
main.log.error( self.name + ": Failed to secure ssh on " + node )
- main.log.debug( self.handle.before )
+ main.log.debug( self.name + ": " + self.handle.before )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -1755,7 +1765,7 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
- def dumpONOSCmd( self, ONOSIp, CMD, destDir, filename, options="", timeout=60 ):
+ def dumpONOSCmd( self, ONOSIp, CMD, destDir, filename, options="", cliPort=8101, timeout=60 ):
"""
Dump Cmd to a desired directory.
For debugging purposes, you may want to use
@@ -1780,7 +1790,7 @@
if destDir[ -1: ] != "/":
destDir += "/"
cmd = CMD + " " + options + " > " + str( destDir ) + str( filename ) + localtime
- return self.onosCli( ONOSIp, cmd, timeout=timeout )
+ return self.onosCli( ONOSIp, cmd, cliPort=cliPort, timeout=timeout )
def cpLogsToDir( self, logToCopy,
destDir, copyFileName="" ):
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 3083f44..aa1e9f9 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -22,6 +22,7 @@
"""
import pexpect
import re
+import os
from drivers.component import Component
@@ -47,6 +48,7 @@
It will take user_name ,ip_address and password as arguments<br>
and will return the handle.
"""
+ self.shell = "/bin/bash -l"
for key in connectargs:
vars( self )[ key ] = connectargs[ key ]
self.checkPrompt()
@@ -55,27 +57,24 @@
ssh_newkey = 'Are you sure you want to continue connecting'
refused = "ssh: connect to host " + \
self.ip_address + " port 22: Connection refused"
+ ssh_options = "-t -X -A -o ServerAliveInterval=120 -o TCPKeepAlive=yes"
+ ssh_destination = self.user_name + "@" + self.ip_address
+ envVars = { "TERM": "vt100" }
+ # TODO: Add option to specify which shell/command to use
+ jump_host = main.componentDictionary[ self.name ].get( 'jump_host' )
if self.port:
- self.handle = pexpect.spawn(
- 'ssh -X -p ' +
- self.port +
- ' ' +
- self.user_name +
- '@' +
- self.ip_address +
- ' -o ServerAliveInterval=120 -o TCPKeepAlive=yes',
- env={ "TERM": "vt100" },
- maxread=1000000 )
- else:
- self.handle = pexpect.spawn(
- 'ssh -X ' +
- self.user_name +
- '@' +
- self.ip_address +
- ' -o ServerAliveInterval=120 -o TCPKeepAlive=yes',
- env={ "TERM": "vt100" },
- maxread=1000000,
- timeout=60 )
+ ssh_option += " -p " + self.port
+ if jump_host:
+ jump_host = main.componentDictionary.get( jump_host )
+ ssh_options += " -J %s@%s" % ( jump_host.get( 'user' ), jump_host.get( 'host' ) )
+ ssh_auth = os.getenv('SSH_AUTH_SOCK')
+ if ssh_auth:
+ envVars[ 'SSH_AUTH_SOCK' ] = ssh_auth
+ self.handle = pexpect.spawn(
+ "ssh %s %s %s" % ( ssh_options, ssh_destination, self.shell ),
+ env=envVars,
+ maxread=1000000,
+ timeout=60 )
# set tty window size
self.handle.setwinsize( 24, 250 )
@@ -119,6 +118,7 @@
return main.FALSE
elif i == 2:
main.log.error( self.name + ": Connection timeout" )
+ main.log.debug( self.handle.before )
return main.FALSE
elif i == 3: # timeout
main.log.error(
@@ -126,15 +126,18 @@
self.user_name +
"@" +
self.ip_address )
+ main.log.debug( self.handle.before )
return main.FALSE
elif i == 4:
main.log.error(
"ssh: connect to host " +
self.ip_address +
" port 22: Connection refused" )
+ main.log.debug( self.handle.before )
return main.FALSE
elif i == 6: # Incorrect Password
main.log.error( self.name + ": Incorrect Password" )
+ main.log.debug( self.handle.before )
return main.FALSE
elif i == 7: # Prompt
main.log.info( self.name + ": Password not required logged in" )
@@ -146,6 +149,7 @@
return self.handle
def disconnect( self ):
+ result = self.preDisconnect()
result = super( CLI, self ).disconnect( self )
result = main.TRUE
@@ -373,6 +377,10 @@
while "to" means copy "to" the remote machine from
local machine
"""
+ jump_host = main.componentDictionary[ remoteHost.name ].get( 'jump_host' )
+ if jump_host:
+ jump_host = main.componentDictionary.get( jump_host )
+ options += " -J %s@%s " % ( jump_host.get( 'user' ), jump_host.get( 'host' ) )
return self.secureCopy( remoteHost.user_name,
remoteHost.ip_address,
filePath,
@@ -728,7 +736,7 @@
main.log.exception( self.name + ": Uncaught exception!" )
return main.FALSE
- def dockerRun( self, image, containerName, options="", imageArgs="" ):
+ def dockerRun( self, image, containerName, options="", imageArgs="", background=False ):
"""
Run a docker image
Required Arguments:
@@ -745,6 +753,8 @@
options if options else "",
image,
imageArgs )
+ if background:
+ cmdStr += " &"
main.log.info( self.name + ": sending: " + cmdStr )
self.handle.sendline( cmdStr)
i = self.handle.expect( [ self.prompt,
@@ -927,3 +937,251 @@
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
return main.FALSE
+
+# TODO: How is this different from exitFromCmd used elsewhere?
+ def exitFromProcess( self ):
+ """
+ Send ctrl-c, which should close and exit the program
+ """
+ try:
+ cmdStr = "\x03"
+ main.log.info( self.name + ": sending: " + repr( cmdStr ) )
+ self.handle.send( cmdStr)
+ i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT ] )
+ if i == 0:
+ return main.TRUE
+ else:
+ main.log.error( self.name + ": Error exiting process" )
+ main.log.debug( self.name + ": " + self.handle.before + str( self.handle.after ) )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ return main.FALSE
+
+ def preDisconnect( self ):
+ """
+ A Stub for a function that will be called before disconnect.
+ This can be set if for instance, the shell is running a program
+ and needs to exit the program before disconnecting from the component
+ """
+ print "preDisconnect"
+ return main.TRUE
+
+ def kubectlGetPodNames( self, kubeconfig=None, namespace=None, app=None, name=None ):
+ """
+ Use kubectl to get the names of pods
+ Optional Arguments:
+ - kubeconfig: The path to a kubeconfig file
+ - namespace: The namespace to search in
+ - app: Get pods belonging to a specific app
+ - name: Get pods with a specific name label
+ Returns a list containing the names of the pods or
+ main.FALSE on Error
+ """
+
+ try:
+ cmdStr = "kubectl %s %s get pods %s %s --output=jsonpath='{.items..metadata.name}{\"\\n\"}'" % (
+ "--kubeconfig %s" % kubeconfig if kubeconfig else "",
+ "-n %s" % namespace if namespace else "",
+ "-l app=%s" % app if app else "",
+ "-l name=%s" % name if name else "" )
+ main.log.info( self.name + ": sending: " + repr( cmdStr ) )
+ self.handle.sendline( cmdStr )
+ i = self.handle.expect( [ "not found", "error", "The connection to the server", self.prompt ] )
+ if i == 3:
+ output = self.handle.before + self.handle.after
+ names = output.split( '\r\n' )[1].split()
+ return names
+ else:
+ main.log.error( self.name + ": Error executing command" )
+ main.log.debug( self.name + ": " + self.handle.before + str( self.handle.after ) )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ return main.FALSE
+
+ def kubectlDescribe( self, describeString, dstPath, kubeconfig=None, namespace=None ):
+ """
+ Use kubectl to get the logs from a pod
+ Required Arguments:
+ - describeString: The string passed to the cli. Example: "pods"
+ - dstPath: The location to save the logs to
+ Optional Arguments:
+ - kubeconfig: The path to a kubeconfig file
+ - namespace: The namespace to search in
+ Returns main.TRUE or
+ main.FALSE on Error
+ """
+
+ try:
+ cmdStr = "kubectl %s %s describe %s > %s " % (
+ "--kubeconfig %s" % kubeconfig if kubeconfig else "",
+ "-n %s" % namespace if namespace else "",
+ describeString,
+ dstPath )
+ main.log.info( self.name + ": sending: " + repr( cmdStr ) )
+ self.handle.sendline( cmdStr )
+ i = self.handle.expect( [ "not found", "error", "The connection to the server", self.prompt ] )
+ if i == 3:
+ main.log.debug( self.name + ": " + self.handle.before )
+ return main.TRUE
+ else:
+ main.log.error( self.name + ": Error executing command" )
+ main.log.debug( self.name + ": " + self.handle.before + str( self.handle.after ) )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ return main.FALSE
+
+ def kubectlPodNodes( self, dstPath=None, kubeconfig=None, namespace=None ):
+ """
+ Use kubectl to get the logs from a pod
+ Optional Arguments:
+ - dstPath: The location to save the logs to
+ - kubeconfig: The path to a kubeconfig file
+ - namespace: The namespace to search in
+ Returns main.TRUE if dstPath is given, else the output of the command or
+ main.FALSE on Error
+ """
+
+ try:
+ cmdStr = "kubectl %s %s get pods -o=custom-columns=NAME:.metadata.name,NODE:.spec.nodeName %s " % (
+ "--kubeconfig %s" % kubeconfig if kubeconfig else "",
+ "-n %s" % namespace if namespace else "",
+ " > %s" % dstPath if dstPath else "" )
+ main.log.info( self.name + ": sending: " + repr( cmdStr ) )
+ self.handle.sendline( cmdStr )
+ i = self.handle.expect( [ "not found", "error", "The connection to the server", self.prompt ] )
+ if i == 3:
+ output = self.handle.before
+ main.log.debug( self.name + ": " + output )
+ return output if dstPath else main.TRUE
+ else:
+ main.log.error( self.name + ": Error executing command" )
+ main.log.debug( self.name + ": " + self.handle.before + str( self.handle.after ) )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ return main.FALSE
+
+ def kubectlLogs( self, podName, dstPath, kubeconfig=None, namespace=None, timeout=240 ):
+ """
+ Use kubectl to get the logs from a pod
+ Required Arguments:
+ - podName: The name of the pod to get the logs of
+ - dstPath: The location to save the logs to
+ Optional Arguments:
+ - kubeconfig: The path to a kubeconfig file
+ - namespace: The namespace to search in
+ - timeout: Timeout for command to return. The longer the logs, the longer it will take to fetch them.
+ Returns main.TRUE or
+ main.FALSE on Error
+ """
+
+ try:
+ cmdStr = "kubectl %s %s logs %s > %s " % (
+ "--kubeconfig %s" % kubeconfig if kubeconfig else "",
+ "-n %s" % namespace if namespace else "",
+ podName,
+ dstPath )
+ main.log.info( self.name + ": sending: " + repr( cmdStr ) )
+ self.handle.sendline( cmdStr )
+ i = self.handle.expect( [ "not found", "error", "The connection to the server", self.prompt ], timeout=timeout )
+ if i == 3:
+ main.log.debug( self.name + ": " + self.handle.before )
+ return main.TRUE
+ else:
+ main.log.error( self.name + ": Error executing command" )
+ main.log.debug( self.name + ": " + self.handle.before + str( self.handle.after ) )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ return main.FALSE
+
+ def kubectlPortForward( self, podName, portsList, kubeconfig=None, namespace=None, ):
+ """
+ Use kubectl to setup port forwarding from the local machine to the kubernetes pod
+
+ Note: This command does not return until the port forwarding session is ended.
+
+ Required Arguments:
+ - podName: The name of the pod as a string
+ - portsList: The list of ports to forward, as a string. see kubectl help for details
+ Optional Arguments:
+ - kubeconfig: The path to a kubeconfig file
+ - namespace: The namespace to search in
+ - app: Get pods belonging to a specific app
+ Returns a list containing the names of the pods or
+ main.FALSE on Error
+
+
+ """
+ try:
+ cmdStr = "kubectl %s %s port-forward pod/%s %s" % (
+ "--kubeconfig %s" % kubeconfig if kubeconfig else "",
+ "-n %s" % namespace if namespace else "",
+ podName,
+ portsList )
+ main.log.info( self.name + ": sending: " + repr( cmdStr ) )
+ self.handle.sendline( cmdStr )
+ i = self.handle.expect( [ "not found", "error", "closed/timedout",
+ self.prompt, "The connection to the server", "Forwarding from" ] )
+ # NOTE: This won't clear the buffer entirely, and each time the port forward
+ # is used, another line will be added to the buffer. We need to make
+ # sure we clear the buffer before using this component again.
+
+ if i == 5:
+ # Setup preDisconnect function
+ self.preDisconnect = self.exitFromProcess
+ return main.TRUE
+ else:
+ main.log.error( self.name + ": Error executing command" )
+ main.log.debug( self.name + ": " + self.handle.before + str( self.handle.after ) )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ return main.FALSE