Allow use of onos docker for existing tests
- Allow Cluster to pull/build onos docker
- Connect clidriver to cli runnning in docker
- Some changes for debugability in components
- To use, set the useDocker and diffCliHosts tags in the cluster
component to True, then define parameters in the params file
- Update all SR Stratum tests to use the tost docker image
- NOTE: Since the tost-onos image doesn't have openflow installe, we are
currently only using docker for the bmv2 and tofino switches
Change-Id: If900b0bdbf9a41b8885c692ccba18a3b1bc580cc
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 682760a..47123a4 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -163,7 +163,7 @@
main.log.info( self.name + ": Sending sudo password" )
self.handle.sendline( self.pwd )
i = self.handle.expect( [ '%s:' % self.user_name,
- self.prompt,
+ self.Prompt(),
pexpect.EOF,
pexpect.TIMEOUT ],
timeout )
@@ -210,12 +210,12 @@
cmdString += mnCmd
# Send the command and check if network started
self.handle.sendline( "" )
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
main.log.info( "Sending '" + cmdString + "' to " + self.name )
self.handle.sendline( cmdString )
startTime = time.time()
while True:
- i = self.handle.expect( [ 'mininet>',
+ i = self.handle.expect( [ self.mnPrompt,
'Exception|Error',
'\*\*\*',
pexpect.EOF,
@@ -228,7 +228,7 @@
elif i == 1:
response = str( self.handle.before +
self.handle.after )
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
response += str( self.handle.before +
self.handle.after )
main.log.error(
@@ -1255,7 +1255,7 @@
try:
response = self.execute(
cmd='dump',
- prompt='mininet>',
+ prompt=self.mnPrompt,
timeout=10 )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
@@ -1271,7 +1271,7 @@
try:
response = self.execute(
cmd='intfs',
- prompt='mininet>',
+ prompt=self.mnPrompt,
timeout=10 )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
@@ -1285,7 +1285,7 @@
def net( self ):
main.log.info( self.name + ": List network connections" )
try:
- response = self.execute( cmd='net', prompt='mininet>', timeout=10 )
+ response = self.execute( cmd='net', prompt=self.mnPrompt, timeout=10 )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -1298,7 +1298,7 @@
def links( self, timeout=1000 ):
main.log.info( self.name + ": List network links" )
try:
- response = self.execute( cmd='links', prompt='mininet>',
+ response = self.execute( cmd='links', prompt=self.mnPrompt,
timeout=timeout )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
@@ -1506,7 +1506,7 @@
try:
response = self.execute(
cmd='nodes',
- prompt='mininet>',
+ prompt=self.mnPrompt,
timeout=10 )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
@@ -1522,7 +1522,7 @@
try:
response = self.execute(
cmd='pingpair',
- prompt='mininet>',
+ prompt=self.mnPrompt,
timeout=20 )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
@@ -2253,7 +2253,7 @@
"""
try:
self.handle.sendline( '' )
- i = self.handle.expect( [ 'mininet>', self.hostPrompt, pexpect.EOF, pexpect.TIMEOUT ],
+ i = self.handle.expect( [ self.mnPrompt, self.hostPrompt, pexpect.EOF, pexpect.TIMEOUT ],
timeout=2 )
response = main.TRUE
if i == 0:
@@ -2292,7 +2292,7 @@
try:
self.handle.sendline( "" )
i = self.handle.expect( [ self.mnPrompt,
- self.bashPrompt,
+ self.Prompt(),
pexpect.EOF,
pexpect.TIMEOUT ],
timeout )
@@ -2301,8 +2301,8 @@
startTime = time.time()
self.handle.sendline( "exit" )
exitRet = 1
- while not exitRet:
- exitRet = self.handle.expect( [ self.bashprompt,
+ while exitRet:
+ exitRet = self.handle.expect( [ self.Prompt(),
"Traceback",
"AssertionError",
self.mnPrompt ],
@@ -2315,6 +2315,7 @@
cmd = "sudo " + cmd
self.handle.sendline( cmd )
response = main.TRUE
+ self.handle.expect( self.Prompt() )
elif i == 1:
main.log.info( " Mininet trying to exit while not " +
@@ -2327,21 +2328,21 @@
"TIMEOUT" )
self.handle.sendline( "" )
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
cmd = "killall -9 dhclient dhcpd zebra bgpd"
if self.sudoRequired:
cmd = "sudo " + cmd
self.handle.sendline( cmd )
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
if fileName:
self.handle.sendline( "" )
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
cmd = "kill -9 \`ps -ef | grep \"" + fileName + "\" | grep -v grep | awk '{print $2}'\`"
if self.sudoRequired:
cmd = "sudo " + cmd
self.handle.sendline( cmd )
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
except pexpect.TIMEOUT:
main.log.error( self.name + ": TIMEOUT exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -3898,7 +3899,7 @@
host = self.name
if self.mExecDir:
self.handle.sendline( "cd %s" % self.mExecDir )
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
self.handle.sendline( self.home + "/util/m " + host )
if self.hostHome:
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index c317fb0..7a0641e 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -58,6 +58,7 @@
self.handle = None
self.karafUser = None
self.karafPass = None
+ self.dockerPrompt = None
self.graph = Graph()
super( OnosCliDriver, self ).__init__()
@@ -82,9 +83,12 @@
self.karafUser = self.options[ key ]
elif key == "karaf_password":
self.karafPass = self.options[ key ]
+ elif key == "docker_prompt":
+ self.dockerPrompt = self.options[ key ]
self.home = self.checkOptions( self.home, "~/onos" )
self.karafUser = self.checkOptions( self.karafUser, self.user_name )
self.karafPass = self.checkOptions( self.karafPass, self.pwd )
+ self.dockerPrompt = self.checkOptions( self.dockerPrompt, "~/onos#" )
for key in self.options:
if key == 'onosIp':
@@ -143,9 +147,15 @@
i = self.logout()
if i == main.TRUE:
self.handle.sendline( "" )
- self.handle.expect( self.prompt )
- self.handle.sendline( "exit" )
- self.handle.expect( "closed" )
+ for l in range( 3 ):
+ p = self.handle.expect( [ self.prompt, self.dockerPrompt ] )
+ if p == 1:
+ self.inDocker = False
+ self.handle.sendline( "exit" )
+ j = self.handle.expect( [ "closed", pexpect.TIMEOUT ], timeout=3 )
+ if j == 0:
+ return response
+ response = main.FALSE
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
response = main.FALSE
@@ -156,7 +166,7 @@
main.log.exception( "Exception in disconnect of " + self.name )
response = main.TRUE
except Exception:
- main.log.exception( self.name + ": Connection failed to the host" )
+ main.log.exception( self.name + ": disconnection failed from the host" )
response = main.FALSE
return response
@@ -171,11 +181,14 @@
try:
if self.handle:
self.handle.sendline( "" )
- i = self.handle.expect( [ self.karafPrompt, self.prompt, pexpect.TIMEOUT ],
+ i = self.handle.expect( [ self.karafPrompt,
+ self.Prompt(),
+ pexpect.TIMEOUT ],
+
timeout=10 )
if i == 0: # In ONOS CLI
self.handle.sendline( "logout" )
- j = self.handle.expect( [ self.prompt,
+ j = self.handle.expect( [ self.Prompt(),
"Command not found:",
pexpect.TIMEOUT ] )
if j == 0: # Successfully logged out
@@ -185,7 +198,7 @@
# or the command timed out
self.handle.send( "\x04" ) # send ctrl-d
try:
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
except pexpect.TIMEOUT:
main.log.error( "ONOS did not respond to 'logout' or CTRL-d" )
return main.TRUE
@@ -272,59 +285,49 @@
try:
# Check if we are already in the cli
self.handle.sendline( "" )
- x = self.handle.expect( [
- self.prompt, self.karafPrompt ], commandlineTimeout )
+ x = self.handle.expect( [ self.Prompt(), self.karafPrompt ], commandlineTimeout )
if x == 1:
main.log.info( "ONOS cli is already running" )
return main.TRUE
# Not in CLI so login
- if waitForStart:
+ if self.inDocker:
+ # The Docker does not have all the wrapper scripts
+ startCliCommand = "ssh -p 8101 -o StrictHostKeyChecking=no %s@localhost" % self.karafUser
+ elif waitForStart:
# Wait for onos start ( onos-wait-for-start ) and enter onos cli
- startCliCommand = "onos-wait-for-start "
+ startCliCommand = "onos-wait-for-start " + str( ONOSIp )
else:
- startCliCommand = "onos "
- self.handle.sendline( startCliCommand + str( ONOSIp ) )
- i = self.handle.expect( [
- self.karafPrompt,
- pexpect.TIMEOUT ], onosStartTimeout )
+ startCliCommand = "onos " + str( ONOSIp )
+ self.handle.sendline( startCliCommand )
+ tries = 0
+ while tries < 5:
+ i = self.handle.expect( [
+ self.karafPrompt,
+ "Password:",
+ pexpect.TIMEOUT ], onosStartTimeout )
- if i == 0:
- main.log.info( str( ONOSIp ) + " CLI Started successfully" )
- if karafTimeout: # FIXME: This doesn't look right
- self.handle.sendline(
- "config:property-set -p org.apache.karaf.shell\
- sshIdleTimeout " +
- karafTimeout )
- self.handle.expect( self.prompt )
- self.handle.sendline( startCliCommand + str( ONOSIp ) )
- self.handle.expect( self.karafPrompt )
- main.log.debug( self.handle.before )
- return main.TRUE
- else:
- # If failed, send ctrl+c to process and try again
- main.log.info( "Starting CLI failed. Retrying..." )
- self.handle.send( "\x03" )
- self.handle.sendline( startCliCommand + str( ONOSIp ) )
- i = self.handle.expect( [ self.karafPrompt, pexpect.TIMEOUT ],
- timeout=30 )
if i == 0:
- main.log.info( str( ONOSIp ) + " CLI Started " +
- "successfully after retry attempt" )
+ main.log.info( str( ONOSIp ) + " CLI Started successfully" )
if karafTimeout:
self.handle.sendline(
"config:property-set -p org.apache.karaf.shell\
- sshIdleTimeout " +
+ sshIdleTimeout " +
karafTimeout )
- self.handle.expect( self.prompt )
- self.handle.sendline( startCliCommand + str( ONOSIp ) )
self.handle.expect( self.karafPrompt )
return main.TRUE
+ elif i == 1:
+ main.log.info( str( ONOSIp ) + " CLI asking for password" )
+ main.log.debug( "Sending %s" % self.karafPass )
+ self.handle.sendline( self.karafPass )
else:
- main.log.error( "Connection to CLI " +
- str( ONOSIp ) + " timeout" )
- return main.FALSE
-
+ # If failed, send ctrl+c to process and try again
+ main.log.info( "Starting CLI failed. Retrying..." )
+ self.handle.send( "\x03" )
+ self.handle.sendline( startCliCommand )
+ tries += 1
+ main.log.error( "Connection to CLI " + str( ONOSIp ) + " timeout" )
+ return main.FALSE
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
return None
@@ -339,7 +342,7 @@
def startCellCli( self, karafTimeout="",
commandlineTimeout=10, onosStartTimeout=60 ):
"""
- Start CLI on onos ecll handle.
+ Start CLI on onos cell handle.
karafTimeout is an optional argument. karafTimeout value passed
by user would be used to set the current karaf shell idle timeout.
@@ -357,7 +360,7 @@
try:
self.handle.sendline( "" )
x = self.handle.expect( [
- self.prompt, self.karafPrompt ], commandlineTimeout )
+ self.Prompt(), self.karafPrompt ], commandlineTimeout )
if x == 1:
main.log.info( "ONOS cli is already running" )
@@ -376,10 +379,9 @@
"config:property-set -p org.apache.karaf.shell\
sshIdleTimeout " +
karafTimeout )
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
self.handle.sendline( "/opt/onos/bin/onos" )
self.handle.expect( self.karafPrompt )
- main.log.debug( self.handle.before )
return main.TRUE
else:
# If failed, send ctrl+c to process and try again
@@ -396,7 +398,7 @@
"config:property-set -p org.apache.karaf.shell\
sshIdleTimeout " +
karafTimeout )
- self.handle.expect( self.prompt )
+ self.handle.expect( self.Prompt() )
self.handle.sendline( "/opt/onos/bin/onos" )
self.handle.expect( self.karafPrompt )
return main.TRUE
@@ -476,7 +478,9 @@
try:
# Try to reconnect if disconnected from cli
self.handle.sendline( "" )
- i = self.handle.expect( [ self.karafPrompt, self.prompt, pexpect.TIMEOUT ] )
+ i = self.handle.expect( [ self.karafPrompt,
+ self.Prompt(),
+ pexpect.TIMEOUT ] )
response = self.handle.before
if i == 1:
main.log.error( self.name + ": onos cli session closed. " )
@@ -495,7 +499,7 @@
main.cleanAndExit()
if i == 2:
main.log.warn( "Timeout when testing cli responsiveness" )
- main.log.debug( self.handle.before )
+ main.log.debug( "%s: %s" % ( self.name, self.handle.before ) )
self.handle.send( "\x03" ) # Send ctrl-c to clear previous output
self.handle.expect( self.karafPrompt )
@@ -505,7 +509,7 @@
main.log.debug( self.name + ": " + repr( response ) )
except pexpect.TIMEOUT:
main.log.error( self.name + ": ONOS timeout" )
- main.log.debug( self.handle.before )
+ main.log.debug( "%s: %s" % ( self.name, self.handle.before ) )
self.handle.send( "\x03" )
self.handle.expect( self.karafPrompt )
return None
@@ -558,8 +562,9 @@
main.log.debug( self.name + ": Raw output" )
main.log.debug( self.name + ": " + repr( response ) )
+ response = self.cleanOutput( response, debug=debug )
# Remove control codes from karaf 4.2.1
- karafEscape = re.compile( r"('(0|1)~\'|\r\r\r\n\x1b\[A\x1b\[79C(x|\s)?|\x1b(>|=)|\x1b\[90m~)" )
+ karafEscape = re.compile( r"('(0|1)~\'|\r\r\r\n\x1b\[A\x1b\[79C(x|\s)?|\x1b(>|=~?)|\x1b\[90m~)" )
response = karafEscape.sub( '', response )
if debug:
main.log.debug( self.name + ": karafEscape output" )
@@ -570,11 +575,15 @@
# Remove ANSI color control strings from output
# NOTE: karaf is sometimes adding a single character then two
# backspaces and sometimes adding 2 characters with 2 backspaces??
- backspaceEscape = re.compile( r'((..\x08\x08)|(.|\s)\x08)' )
- response = backspaceEscape.sub( '', response )
- if debug:
- main.log.debug( self.name + ": backspaceEscape output" )
- main.log.debug( self.name + ": " + repr( response ) )
+ backspaceEscape = re.compile( r'((.|\s)\x08)' )
+ unchanged = False
+ while not unchanged:
+ old = response
+ response = backspaceEscape.sub( '', response, count=1 )
+ if debug:
+ main.log.debug( self.name + ": backspaceEscape output" )
+ main.log.debug( self.name + ": " + repr( response ) )
+ unchanged = old == response
# Remove extra return chars that get added
response = re.sub( r"\s\r", "", response )
@@ -632,7 +641,7 @@
except pexpect.TIMEOUT:
main.log.error( self.name + ": ONOS timeout" )
if debug:
- main.log.debug( self.handle.before )
+ main.log.debug( "%s: %s" % ( self.name, self.handle.before ) )
self.exitFromCmd( self.karafPrompt, 100 )
return None
except IndexError:
@@ -6713,3 +6722,100 @@
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+
+ def prepareForCLI( self, debug=True, maxRetries=120 ):
+ """
+ Prepare docker container to connect to onos cli
+ """
+ try:
+ # Wait for log files to be created
+ ready = 0
+ retries = 0
+ while not ready and retries < maxRetries:
+ retries += 1
+ self.handle.sendline( "ls -al /root/onos/apache-karaf-*/data/log" )
+ ready = self.handle.expect( [ "No such file or directory", self.dockerPrompt ] )
+ main.log.debug( "%s: %s" % ( self.name, self.handle.before ) )
+ if not ready:
+ self.handle.expect( self.dockerPrompt )
+ time.sleep( 1 )
+ #main.log.debug( "%s: It took %s tries for onos log folder to %sbe created" %
+ # ( self.name, retries, "" if ready else "NOT " ) )
+
+ cmdList = []
+ cmdList.append( "apt-get update" )
+ cmdList.append( "apt-get install -y openssh-server" )
+ # Some built in scripts are hardcoded
+ cmdList.append( "ln -s /root/onos /opt/onos" )
+ cmdList.append( "ln -s /root/onos/apache-karaf-*/data/log /opt/onos/log" )
+ cmdList.append( "ls -al /opt/onos" )
+ output = ""
+ for cmdStr in cmdList:
+ self.handle.sendline( cmdStr )
+ self.handle.expect( self.dockerPrompt )
+ self.handle.sendline( "" )
+ self.handle.expect( self.dockerPrompt )
+ handle = self.handle.before
+ assert "command not found" not in handle, handle
+ assert "No such file or directory" not in handle, handle
+ output += handle
+ if debug:
+ main.log.debug( "%s: %s" % ( self.name, output ) )
+ return output
+ except AssertionError:
+ main.log.exception( "" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
+ def onosSecureSSH( self, userName="onos", userPWD="rocks" ):
+ """
+ Enables secure access to ONOS console
+ by removing default users & keys.
+
+ bin/onos-user-password onos rocks
+
+ Returns: main.TRUE on success and main.FALSE on failure
+ """
+
+ try:
+ self.handle.sendline( "" )
+ self.handle.expect( self.dockerPrompt )
+
+ self.handle.sendline( "[ ! -f ~/.ssh/id_rsa.pub ] && ssh-keygen -t rsa -f ~/.ssh/id_rsa -P '' -q" )
+ self.handle.expect( self.dockerPrompt )
+ main.log.debug( "%s: %s%s" % ( self.name, self.handle.before, self.handle.after ) )
+
+ self.handle.sendline( "bin/onos-user-key $(id -un) $(cut -d\\\\ -f2 ~/.ssh/id_rsa.pub)" )
+ self.handle.expect( pexpect.TIMEOUT, timeout=10 )
+ main.log.debug( "%s: %s" % ( self.name, self.handle.before ) )
+
+ self.handle.sendline( "bin/onos-user-password %s %s" % ( userName, userPWD ) )
+ i = self.handle.expect( [ "usage",
+ self.dockerPrompt,
+ pexpect.TIMEOUT ] )
+ if i == 0:
+ # malformed command
+ main.log.warn( self.name + ": Could not parse onos-user-password command" )
+ self.handle.expect( self.dockerPrompt )
+ return main.FALSE
+ elif i == 1:
+ # Process started
+ main.log.info( self.name + ": SSH password added for user " + userName )
+ return main.TRUE
+ elif i == 2:
+ # timeout
+ main.log.error( self.name + ": Failed to secure onos ssh " )
+ main.log.debug( "%s: %s" % ( self.name, self.handle.before ) )
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
diff --git a/TestON/drivers/common/cli/onosclusterdriver.py b/TestON/drivers/common/cli/onosclusterdriver.py
index 83d4295..520250b 100755
--- a/TestON/drivers/common/cli/onosclusterdriver.py
+++ b/TestON/drivers/common/cli/onosclusterdriver.py
@@ -56,27 +56,19 @@
We will look into each of the node's component handles to try to find the attreibute, looking at REST first
"""
- usedDriver = False
if hasattr( self.REST, name ):
- if not usedDriver:
- usedDriver = True
- main.log.debug( "%s: Using Rest driver's attribute for '%s'" % ( self.name, name ) )
- f = getattr( self.REST, name )
+ main.log.debug( "%s: Using Rest driver's attribute for '%s'" % ( self.name, name ) )
+ return getattr( self.REST, name )
if hasattr( self.CLI, name ):
- if not usedDriver:
- usedDriver = True
- main.log.debug( "%s: Using CLI driver's attribute for '%s'" % ( self.name, name ) )
- f = getattr( self.CLI, name )
+ main.log.debug( "%s: Using CLI driver's attribute for '%s'" % ( self.name, name ) )
+ return getattr( self.CLI, name )
if hasattr( self.Bench, name ):
- if not usedDriver:
- usedDriver = True
- main.log.debug( "%s: Using Bench driver's attribute for '%s'" % ( self.name, name ) )
- f = getattr( self.Bench, name )
- if usedDriver:
- return f
+ main.log.debug( "%s: Using Bench driver's attribute for '%s'" % ( self.name, name ) )
+ return getattr( self.Bench, name )
raise AttributeError( "Could not find the attribute %s in %r or it's component handles" % ( name, self ) )
- def __init__( self, name, ipAddress, CLI=None, REST=None, Bench=None, pos=None, userName=None, server=None ):
+ def __init__( self, name, ipAddress, CLI=None, REST=None, Bench=None, pos=None,
+ userName=None, server=None, dockerPrompt=None ):
# TODO: validate these arguments
self.name = str( name )
self.ipAddress = ipAddress
@@ -88,6 +80,7 @@
self.ip_address = ipAddress
self.user_name = userName
self.server = server
+ self.dockerPrompt = dockerPrompt
class OnosClusterDriver( CLI ):
@@ -98,6 +91,8 @@
self.name = None
self.home = None
self.handle = None
+ self.useDocker = False
+ self.dockerPrompt = None
self.nodes = []
super( OnosClusterDriver, self ).__init__()
@@ -127,11 +122,17 @@
self.karafPass = self.options[ key ]
elif key == "cluster_name":
prefix = self.options[ key ]
+ elif key == "useDocker":
+ self.useDocker = "True" == self.options[ key ]
+ elif key == "docker_prompt":
+ self.dockerPrompt = self.options[ key ]
self.home = self.checkOptions( self.home, "~/onos" )
self.karafUser = self.checkOptions( self.karafUser, self.user_name )
self.karafPass = self.checkOptions( self.karafPass, self.pwd )
prefix = self.checkOptions( prefix, "ONOS" )
+ self.useDocker = self.checkOptions( self.useDocker, False )
+ self.dockerPrompt = self.checkOptions( self.dockerPrompt, "~/onos#" )
self.name = self.options[ 'name' ]
@@ -435,4 +436,6 @@
rest = self.createRestComponent( restName, ip )
bench = self.createBenchComponent( benchName )
server = self.createServerComponent( serverName, ip ) if createServer else None
- self.nodes.append( Controller( prefix + str( i ), ip, cli, rest, bench, i - 1, self.user_name, server=server ) )
+ self.nodes.append( Controller( prefix + str( i ), ip, cli, rest, bench, i - 1,
+ self.user_name, server=server,
+ dockerPrompt=self.dockerPrompt ) )
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index a0a8dd1..6b13e3a 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -96,7 +96,7 @@
break
if not self.onosIps:
- main.log.info( "Could not read any environment variable"
+ main.log.info( self.name + ": Could not read any environment variable"
+ " please load a cell file with all" +
" onos IP" )
self.maxNodes = None
@@ -105,7 +105,7 @@
str( self.onosIps.values() ) +
" ONOS IPs" )
except KeyError:
- main.log.info( "Invalid environment variable" )
+ main.log.info( self.name + ": Invalid environment variable" )
except Exception as inst:
main.log.error( "Uncaught exception: " + str( inst ) )
@@ -117,7 +117,7 @@
": Trying to connect to " +
self.ip_address )
except KeyError:
- main.log.info( "Invalid host name," +
+ main.log.info( self.name + ": Invalid host name," +
" connecting to local host instead" )
self.ip_address = 'localhost'
except Exception as inst:
@@ -136,7 +136,7 @@
self.handle.expect( self.prompt )
return self.handle
else:
- main.log.info( "Failed to create ONOS handle" )
+ main.log.info( self.name + ": Failed to create ONOS handle" )
return main.FALSE
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
@@ -224,7 +224,7 @@
elif i == 4:
# Prompt returned
break
- main.log.info( "onos-package command returned: " + handle )
+ main.log.info( self.name + ": onos-package command returned: " + handle )
# As long as the sendline does not time out,
# return true. However, be careful to interpret
# the results of the onos-package command return
@@ -258,7 +258,7 @@
handle = str( self.handle.before )
self.handle.expect( self.prompt )
- main.log.info( "onos-build command returned: " +
+ main.log.info( self.name + ": onos-build command returned: " +
handle )
if i == 0:
@@ -285,7 +285,7 @@
On Failure, exits the test
"""
try:
- main.log.info( "Running 'mvn clean install' on " +
+ main.log.info( self.name + ": Running 'mvn clean install' on " +
str( self.name ) +
". This may take some time." )
self.handle.sendline( "cd " + self.home )
@@ -643,7 +643,7 @@
self.name +
": Git Checkout %s : Already on this branch" % branch )
self.handle.expect( self.prompt )
- # main.log.info( "DEBUG: after checkout cmd = "+
+ # main.log.info( self.name + ": DEBUG: after checkout cmd = "+
# self.handle.before )
return main.TRUE
elif i == 3:
@@ -651,7 +651,7 @@
self.name +
": Git checkout %s - Switched to this branch" % branch )
self.handle.expect( self.prompt )
- # main.log.info( "DEBUG: after checkout cmd = "+
+ # main.log.info( self.name + ": DEBUG: after checkout cmd = "+
# self.handle.before )
return main.TRUE
elif i == 4:
@@ -948,7 +948,7 @@
self.handle.expect( self.prompt )
handleBefore = self.handle.before
handleAfter = self.handle.after
- main.log.info( "Verify cell returned: " + handleBefore +
+ main.log.info( self.name + ": Verify cell returned: " + handleBefore +
handleAfter )
return main.TRUE
except pexpect.ExceptionPexpect:
@@ -989,7 +989,7 @@
self.handle.expect( ":~" )
if "value=" + paramValue + "," in self.handle.before:
- main.log.info( "cfg " + configName + " successfully set to " + configParam )
+ main.log.info( self.name + ": cfg " + configName + " successfully set to " + configParam )
return main.TRUE
except pexpect.ExceptionPexpect:
main.log.exception( self.name + ": Pexpect exception found: " )
@@ -1030,13 +1030,19 @@
self.handle.expect( self.prompt )
self.handle.sendline( "onos-wait-for-start " + ONOSIp )
- self.handle.expect( self.prompt )
+ i = self.handle.expect( [ self.prompt, "Password: " ] )
+ if i == 1:
+ self.handle.sendline( self.pwd )
+ self.handle.expect( self.prompt )
- self.handle.sendline( "onos " + ONOSIp + " " + cmdstr )
- i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT ], timeout=timeout )
+ self.handle.sendline( "ssh -q -p 8101 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s %s " % ( ONOSIp, cmdstr ) )
+ i = self.handle.expect( [ self.prompt, "Password: ", pexpect.TIMEOUT ], timeout=timeout )
+ if i == 1:
+ self.handle.sendline( self.pwd )
+ i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT ], timeout=timeout )
if i == 0:
handleBefore = self.handle.before
- main.log.info( "Command sent successfully" )
+ main.log.info( self.name + ": Command sent successfully" )
# Obtain return handle that consists of result from
# the onos command. The string may need to be
# configured further.
@@ -1044,6 +1050,7 @@
return returnString
elif i == 1:
main.log.error( self.name + ": Timeout when sending " + cmdstr )
+ main.log.debug( self.handle.before )
self.handle.send( "\x03" ) # Control-C
self.handle.expect( self.prompt )
return main.FALSE
@@ -1140,7 +1147,7 @@
return main.TRUE
elif i == 2:
# same bits are already on ONOS node
- main.log.info( "ONOS is already installed on " + node )
+ main.log.info( self.name + ": ONOS is already installed on " + node )
self.handle.expect( self.prompt )
return main.TRUE
elif i == 3:
@@ -1150,7 +1157,7 @@
return main.FALSE
elif i == 4:
# prompt
- main.log.info( "ONOS was installed on {} {}.".format( node,
+ main.log.info( self.name + ": ONOS was installed on {} {}.".format( node,
"but not started" if 'n' in options else "and started" ) )
return main.TRUE
elif i == 5:
@@ -1190,14 +1197,14 @@
pexpect.TIMEOUT ], timeout=180 )
if i == 0:
self.handle.expect( self.prompt )
- main.log.info( "Service is already running" )
+ main.log.info( self.name + ": Service is already running" )
return main.TRUE
elif i == 1:
self.handle.expect( self.prompt )
- main.log.info( "ONOS service started" )
+ main.log.info( self.name + ": ONOS service started" )
return main.TRUE
elif i == 2:
- main.log.info( "ONOS service started" )
+ main.log.info( self.name + ": ONOS service started" )
return main.TRUE
else:
self.handle.expect( self.prompt )
@@ -1230,11 +1237,11 @@
pexpect.TIMEOUT ], timeout=180 )
if i == 0:
self.handle.expect( self.prompt )
- main.log.info( "ONOS service stopped" )
+ main.log.info( self.name + ": ONOS service stopped" )
return main.TRUE
elif i == 1:
self.handle.expect( self.prompt )
- main.log.info( "onosStop() Unknown ONOS instance specified: " +
+ main.log.info( self.name + ": onosStop() Unknown ONOS instance specified: " +
str( nodeIp ) )
return main.FALSE
elif i == 2:
@@ -1242,7 +1249,7 @@
main.log.warn( "ONOS wasn't running" )
return main.TRUE
elif i == 3:
- main.log.info( "ONOS service stopped" )
+ main.log.info( self.name + ": ONOS service stopped" )
return main.TRUE
else:
main.log.error( "ONOS service failed to stop" )
@@ -1266,7 +1273,7 @@
self.handle.expect( self.prompt, timeout=180 )
self.handle.sendline( "onos-uninstall " + str( nodeIp ) )
self.handle.expect( self.prompt, timeout=180 )
- main.log.info( "ONOS " + nodeIp + " was uninstalled" )
+ main.log.info( self.name + ": ONOS " + nodeIp + " was uninstalled" )
# onos-uninstall command does not return any text
return main.TRUE
except pexpect.TIMEOUT:
@@ -1297,13 +1304,13 @@
"ONOS\sprocess\sis\snot\srunning",
pexpect.TIMEOUT ], timeout=60 )
if i == 0:
- main.log.info( "ONOS instance " + str( nodeIp ) +
+ main.log.info( self.name + ": ONOS instance " + str( nodeIp ) +
" was killed and stopped" )
self.handle.sendline( "" )
self.handle.expect( self.prompt )
return main.TRUE
elif i == 1:
- main.log.info( "ONOS process was not running" )
+ main.log.info( self.name + ": ONOS process was not running" )
self.handle.sendline( "" )
self.handle.expect( self.prompt )
return main.FALSE
@@ -1337,7 +1344,7 @@
nodeIp ) + " was killed" )
return main.TRUE
elif i == 1:
- main.log.info( "No route to host" )
+ main.log.info( self.name + ": No route to host" )
return main.FALSE
elif i == 2:
main.log.info(
@@ -1346,7 +1353,7 @@
" not configured" )
return main.FALSE
else:
- main.log.info( "ONOS instance was not killed" )
+ main.log.info( self.name + ": ONOS instance was not killed" )
return main.FALSE
except pexpect.EOF:
@@ -1359,15 +1366,19 @@
def onosAppInstall( self, nodeIp, oarFile ):
"""
- Calls the command: 'onos-app nodeIp install! oarFile'
+ Calls the command: 'onos-app nodeIp reinstall! oarFile'
Installs an ONOS application from an oar file
"""
try:
- cmd = "onos-app " + str( nodeIp ) +" install! " + str(oarFile)
+ cmd = "onos-app " + str( nodeIp ) + " reinstall! " + str( oarFile )
self.handle.sendline( cmd )
- self.handle.expect( self.prompt )
+ i = self.handle.expect( [ "409 Conflict", self.prompt ] )
+ if i == 0:
+ self.handle.expect( self.prompt )
+ time.sleep( 30 )
+ self.handle.sendline( cmd )
handle = self.handle.before
- main.log.debug( handle )
+ main.log.debug( "%s: %s" % ( self.name, handle ) )
assert handle is not None, "Error in sendline"
assert "Command not found:" not in handle, handle
assert "error" not in handle, handle
@@ -1442,7 +1453,7 @@
self.handle.sendline( "onos-start-network " + mntopo )
self.handle.expect( "mininet>" )
- main.log.info( "Network started, entered mininet prompt" )
+ main.log.info( self.name + ": Network started, entered mininet prompt" )
# TODO: Think about whether return is necessary or not
@@ -1557,7 +1568,7 @@
sendCmd = addApp + " > " + str( dirFile ) + " &"
else:
sendCmd = addApp + " &"
- main.log.info( "Send cmd: " + sendCmd )
+ main.log.info( self.name + ": Send cmd: " + sendCmd )
self.handle.sendline( sendCmd )
@@ -1588,7 +1599,7 @@
self.handle.sendline( "\n" )
self.handle.expect( self.prompt )
- main.log.info( "Tshark started capturing files on " +
+ main.log.info( self.name + ": Tshark started capturing files on " +
str( interface ) + " and saving to directory: " +
str( dirFile ) )
except pexpect.EOF:
@@ -1687,7 +1698,7 @@
self.handle.sendline( "sudo kill -9 `ps -ef | grep \"tshark -i\"" +
" | grep -v grep | awk '{print $2}'`" )
self.handle.sendline( "" )
- main.log.info( "Tshark stopped" )
+ main.log.info( self.name + ": Tshark stopped" )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -1713,7 +1724,7 @@
if i == 0:
handle = self.handle.before
- main.log.info( "ptpd returned an error: " +
+ main.log.info( self.name + ": ptpd returned an error: " +
str( handle ) )
return handle
elif i == 1:
@@ -1856,10 +1867,10 @@
self.handle.expect( self.prompt )
if i == 0 or i == 1:
- main.log.info( "ONOS is running" )
+ main.log.info( self.name + ": ONOS is running" )
return main.TRUE
elif i == 2 or i == 3:
- main.log.info( "ONOS is stopped" )
+ main.log.info( self.name + ": ONOS is stopped" )
main.log.error( "ONOS service failed to check the status" )
main.cleanAndExit()
@@ -2033,7 +2044,7 @@
deviceCount - number of switches to be assigned
'''
- main.log.info( "Creating link graph configuration file." )
+ main.log.info( self.name + ": Creating link graph configuration file." )
linkGraphPath = self.home + "/tools/package/etc/linkGraph.cfg"
tempFile = "/tmp/linkGraph.cfg"
@@ -2056,7 +2067,7 @@
switchList[ node ] += 1
if isinstance( deviceCount, list ):
- main.log.info( "Using provided device distribution" )
+ main.log.info( self.name + ": Using provided device distribution" )
switchList = [ 0 ]
for i in deviceCount:
switchList.append( int( i ) )
@@ -2099,7 +2110,7 @@
# SCP
os.system( "scp " + tempFile + " " + self.user_name + "@" + benchIp + ":" + linkGraphPath )
- main.log.info( "linkGraph.cfg creation complete" )
+ main.log.info( self.name + ": linkGraph.cfg creation complete" )
def configNullDev( self, ONOSIpList, deviceCount, numPorts=10 ):
'''
@@ -2108,13 +2119,13 @@
numPorts = number of ports per device. Defaults to 10 both in this function and in ONOS. Optional arg
'''
- main.log.info( "Configuring Null Device Provider" )
+ main.log.info( self.name + ": Configuring Null Device Provider" )
clusterCount = len( ONOSIpList )
try:
if isinstance( deviceCount, int ) or isinstance( deviceCount, str ):
- main.log.info( "Creating device distribution" )
+ main.log.info( self.name + ": Creating device distribution" )
deviceCount = int( deviceCount )
switchList = [ 0 ]*( clusterCount+1 )
baselineSwitchCount = deviceCount/clusterCount
@@ -2126,7 +2137,7 @@
switchList[ node ] += 1
if isinstance( deviceCount, list ):
- main.log.info( "Using provided device distribution" )
+ main.log.info( self.name + ": Using provided device distribution" )
if len( deviceCount ) == clusterCount:
switchList = [ '0' ]
@@ -2207,7 +2218,7 @@
main.log.error( self.name + ": " + self.handle.before )
main.cleanAndExit()
except AssertionError:
- main.log.info( "Settings did not post to ONOS" )
+ main.log.info( self.name + ": Settings did not post to ONOS" )
main.log.error( verification )
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
@@ -2270,7 +2281,7 @@
the end point for extraction of data
"""
try:
- main.log.info( " Log Report for {} ".format( nodeIp ).center( 70, '=' ) )
+ main.log.info( self.name + ": Log Report for {} ".format( nodeIp ).center( 70, '=' ) )
if isinstance( searchTerms, str ):
searchTerms = [ searchTerms ]
numTerms = len( searchTerms )
@@ -2298,7 +2309,7 @@
count += 1
if before.index( line ) > ( len( before ) - 7 ):
logLines[ termIndex ].append( line )
- main.log.info( "{}: {}".format( term, count ) )
+ main.log.info( self.name + ": {}: {}".format( term, count ) )
totalHits += count
if termIndex == numTerms - 1:
print "\n"
@@ -2310,7 +2321,7 @@
outputString += ( "\t" + term[ line ] + "\n" )
if outputString != ( term[ 0 ] + ": \n" ):
main.log.info( outputString )
- main.log.info( "=" * 70 )
+ main.log.info( self.name + ": =" * 70 )
return totalHits
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
@@ -2551,7 +2562,7 @@
return main.TRUE if onosStatus else main.FALSE
- def onosNetCfg( self, controllerIp, path, fileName ):
+ def onosNetCfg( self, controllerIp, path, fileName, user=None, password=None):
"""
Push a specified json file to ONOS through the onos-netcfg service
@@ -2564,14 +2575,23 @@
there is an error.
"""
try:
- cmd = "onos-netcfg {0} {1}{2}".format( controllerIp, path, fileName )
- main.log.info( "Sending: " + cmd )
- main.ONOSbench.handle.sendline( cmd )
- main.ONOSbench.handle.expect( self.prompt )
+ cmd = "onos-netcfg "
+ if user:
+ cmd += "-u %s " % user
+ if password:
+ cmd += "-p %s " % password
+ cmd += "{0} {1}{2}".format( controllerIp, path, fileName )
+ main.log.info( self.name + ": Sending: " + cmd )
+ self.handle.sendline( cmd )
+ self.handle.expect( self.prompt )
handle = self.handle.before
- if "Error" in handle or "No such file or directory" in handle or "curl: " in handle:
+ if "Error" in handle or\
+ "No such file or directory" in handle or\
+ "command not found" in handle or\
+ "curl: " in handle:
main.log.error( self.name + ": " + handle + self.handle.after )
return main.FALSE
+ main.log.debug( self.name + ": " + handle )
return main.TRUE
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
@@ -2588,7 +2608,7 @@
try:
onosIPs = " ".join( onosIPs )
command = "onos-form-cluster {}".format( onosIPs )
- main.log.info( "Sending: " + command )
+ main.log.info( self.name + ": Sending: " + command )
self.handle.sendline( "" )
self.handle.expect( self.prompt )
self.handle.sendline( command )
@@ -2787,16 +2807,16 @@
pexpect.TIMEOUT ], timeout=60 )
if i == 0:
- main.log.info( "Atomix instance " + str( nodeIp ) + " was killed" )
+ main.log.info( self.name + ": Atomix instance " + str( nodeIp ) + " was killed" )
return main.TRUE
elif i == 1:
- main.log.info( "No route to host" )
+ main.log.info( self.name + ": No route to host" )
return main.FALSE
elif i == 2:
- main.log.info( "Passwordless login for host: " + str( nodeIp ) + " not configured" )
+ main.log.info( self.name + ": Passwordless login for host: " + str( nodeIp ) + " not configured" )
return main.FALSE
else:
- main.log.info( "Atomix instance was not killed" )
+ main.log.info( self.name + ": Atomix instance was not killed" )
return main.FALSE
except pexpect.EOF:
@@ -2817,7 +2837,7 @@
self.handle.expect( self.prompt, timeout=180 )
self.handle.sendline( "atomix-uninstall " + str( nodeIp ) )
self.handle.expect( self.prompt, timeout=180 )
- main.log.info( "Atomix " + nodeIp + " was uninstalled" )
+ main.log.info( self.name + ": Atomix " + nodeIp + " was uninstalled" )
# onos-uninstall command does not return any text
return main.TRUE
except pexpect.TIMEOUT:
@@ -2913,9 +2933,9 @@
if dstPath:
cmd += "-P %s " % ( dstPath )
cmd += str( url )
- main.log.info( "Sending: " + cmd )
- main.ONOSbench.handle.sendline( cmd )
- main.ONOSbench.handle.expect( self.prompt )
+ main.log.info( self.name + ": Sending: " + cmd )
+ self.handle.sendline( cmd )
+ self.handle.expect( self.prompt )
output = self.handle.before
main.log.debug( output )
if "Error" in output or "No such file or directory" in output:
@@ -2948,9 +2968,9 @@
# FIXME: Not all options may work, more testing is required, only tested with install(!)
try:
cmd = "onos-app %s %s %s/%s" % ( onosIP, option, filePath, fileName )
- main.log.info( "Sending: " + cmd )
- main.ONOSbench.handle.sendline( cmd )
- main.ONOSbench.handle.expect( self.prompt )
+ main.log.info( self.name + ": Sending: " + cmd )
+ self.handle.sendline( cmd )
+ self.handle.expect( self.prompt )
handle = self.handle.before
main.log.debug( handle )
if "Error" in handle or "usage: " in handle or "curl: " in handle:
@@ -2964,3 +2984,101 @@
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+
+ def makeDocker( self, path, cmd, prompt="Successfully tagged", timeout=600 ):
+ """
+ Build a docker image using a command, such as make
+ Arguments:
+ - path: a path where the script is located. will cd to path
+ - cmd: the command to run
+ Optional Arguments:
+ - prompt: A custom prompt to expect after the command is finished,
+ incase the host prompt is printed during the build
+ - timeout: how long to wait for the build
+ """
+ try:
+ main.log.warn( "%s: makeDocker()" % self.name )
+ self.handle.sendline( "cd %s" % path )
+ self.handle.expect( self.prompt )
+ self.handle.sendline( cmd )
+ self.handle.expect( prompt, timeout=timeout )
+ fullResponse = self.handle.before
+ tailResponse = self.handle.after
+ # TODO: error checking, might be difficult with custom expects
+ self.handle.expect( self.prompt )
+ tailResponse += self.handle.before + self.handle.after
+ fullResponse += tailResponse
+ main.log.debug( self.name + ": " + tailResponse )
+ self.handle.sendline( "cd %s" % self.home )
+ self.handle.expect( self.prompt )
+ return main.TRUE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.log.debug( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+
+ def generateOnosConfig( self, nodeIp, path="cluster.json" ):
+ """
+ Generate onos cluster configuration file
+ Arguments:
+ - nodeIp: ip of the node this file is fore
+ Optional Arguments:
+ - The path to save the file to
+ """
+ try:
+ main.log.info( "%s: Generating onos config file for %s" % ( self.name, nodeIp ) )
+ self.handle.sendline( "onos-gen-config %s %s" % ( nodeIp, path ) )
+ i = self.handle.expect( [ self.prompt, "not found", "Error" ] )
+ response = self.handle.before
+ if i == 0:
+ main.log.debug( "%s: %s" % ( self.name, response ) )
+ return main.TRUE
+ else:
+ response += self.handle.after
+ self.handle.expect( self.prompt )
+ response += self.handle.before
+ main.log.debug( "%s: %s" % ( self.name, response ) )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.log.debug( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+
+ def generateAtomixConfig( self, nodeIp, path="atomix.json" ):
+ """
+ Generate atomix cluster configuration file
+ Arguments:
+ - nodeIp: ip of the node this file is fore
+ Optional Arguments:
+ - The path to save the file to
+ """
+ try:
+ main.log.info( "%s: Generating atomix config file for %s" % ( self.name, nodeIp ) )
+ self.handle.sendline( "atomix-gen-config %s %s" % ( nodeIp, path ) )
+ i = self.handle.expect( [ self.prompt, "not found", "Error" ] )
+ response = self.handle.before
+ if i == 0:
+ main.log.debug( "%s: %s" % ( self.name, response ) )
+ return main.TRUE
+ else:
+ response += self.handle.after
+ self.handle.expect( self.prompt )
+ response += self.handle.before
+ main.log.debug( "%s: %s" % ( self.name, response ) )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.log.debug( self.name + ": " + self.handle.before )
+ main.cleanAndExit()