Merge pull request #130 from opennetworkinglab/devl/pexpect_fix
Devl/pexpect fix
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 832e139..bb30335 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -98,6 +98,27 @@
response = main.FALSE
return response
+ def getEpochMs( self ):
+ """
+ Returns milliseconds since epoch
+
+ When checking multiple nodes in a for loop,
+ around a hundred milliseconds of difference (ascending) is
+ generally acceptable due to calltime of the function.
+ Few seconds, however, is not and it means clocks
+ are off sync.
+ """
+ try:
+ self.handle.sendline( 'date +%s.%N' )
+ self.handle.expect( 'date \+\%s\.\%N' )
+ self.handle.expect( '\$' )
+ epochMs = self.handle.before
+ return epochMs
+ except Exception:
+ main.log.exception( 'Uncaught exception getting epoch time' )
+ main.cleanup()
+ main.exit()
+
def onosPackage( self, opTimeout=30 ):
"""
Produce a self-contained tar.gz file that can be deployed
diff --git a/TestON/tests/FuncPlatform/Dependency/App.py b/TestON/tests/FuncPlatform/Dependency/App.py
index f6fecd6..5410c32 100644
--- a/TestON/tests/FuncPlatform/Dependency/App.py
+++ b/TestON/tests/FuncPlatform/Dependency/App.py
@@ -1,3 +1,8 @@
+"""
+Methods related to application interaction
+
+"""
+
def __init__( self ):
self.ip = '127.0.0.1'
@@ -10,27 +15,48 @@
nodeToActivateFrom = range( 0, nodes )
"""
if isinstance( apps, ( int, basestring ) ):
- main.log.error( "Please pass in a list of strings for args" )
+ main.log.error( 'Please pass in a list of strings for args' )
return main.FALSE
if not isinstance( nodeToActivateFrom, ( int ) ) or \
nodeToActivateFrom < 0:
- main.log.error( "Incorrect node specified" )
+ main.log.error( 'Incorrect node specified' )
return main.FALSE
+ # TODO: Start log capture and listen for exceptions
+ # and errors. Also investigate possible keywords
+ # to listen for when activating applications
+
+
for app in apps:
# Check if app str in appList is in the main scope
# definition main.appList
if app not in main.appList:
- main.log.error( "Invalid app name given" )
+ main.log.error( 'Invalid app name given' )
return main.FALSE
-
- # NOTE: assumes node 1 is always activating application
- appOutput = main.CLIs[nodeToActivateFrom].activateApp(
+
+ try:
+ # NOTE: assumes node 1 is always activating application
+ appOutput = main.CLIs[nodeToActivateFrom].activateApp(
main.appList[app] )
+ except KeyError:
+ main.log.error( 'There was an error with the key '+
+ str(app) + '. Check the appList dictionary' )
+ return main.FALSE
+ except Exception:
+ main.log.error( 'Uncaught exception error while ' +
+ 'activating applications: ' + str(app) )
+ return main.FALSE
return main.TRUE
+def deactivate( apps, nodeToDeactivateFrom=0 ):
+ """
+ Deactivate specified applications from node specified
+
+ """
+ main.log.report( 'deactivate implment me' )
+
def isAppInstallSuccess():
"""
Check the app list across clusters to determine
@@ -38,5 +64,5 @@
"""
- main.log.report( "isAppInstallSuccess" )
+ main.log.report( 'isAppInstallSuccess implement me' )
diff --git a/TestON/tests/FuncPlatform/Dependency/Logger.py b/TestON/tests/FuncPlatform/Dependency/Logger.py
new file mode 100644
index 0000000..22d3f61
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Logger.py
@@ -0,0 +1,57 @@
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def checkOnosLog( nodeIp, option='',
+ outputType=0):
+ """
+ Listens to the log for any Errors and Exceptions.
+
+ Runs 'onos-check-logs <option>'
+ This script only returns if there are any errors
+ or exceptions
+
+ outputType
+ 0: Return output of log
+ 1: Return (#Errors, #Exceptions, #Warn)
+
+ """
+ if not isinstance( option, basestring ):
+ main.log.error( 'Incorrect grep format specified' )
+ return main.FALSE
+
+ try:
+ main.log.info( 'Starting Onos-log listening for '+
+ str(option) )
+ cmd = 'onos-check-logs ' + str(nodeIp) + ' old'
+ if outputType == 0:
+ main.ONOSbench.handle.sendline( cmd )
+ main.ONOSbench.handle.expect( cmd )
+ main.ONOSbench.handle.expect('\$')
+ logResult = main.ONOSbench.handle.before
+ return logResult
+ elif outputType == 1:
+ # Important in assertion criteria
+ # to determine how much warn / error is
+ # acceptable
+ return 'Implement option 1'
+ else:
+ main.log.error( 'Incorrect outputType specified' )
+ return main.FALSE
+
+ except Exception:
+ main.log.exception( self.name + ': Uncaught exception' )
+ main.cleanup()
+ main.exit()
+
+def setLogLevel( level ):
+ """
+ Set the log level of onos
+ """
+ main.log.info( 'setLogLevel implement me' )
+
+def getLogReport( nodeIp, searchTerms ):
+ """
+ Refer to CLI driver for 'logReport'
+ """
+ main.log.info( 'getLogReport - implement me!' )
diff --git a/TestON/tests/FuncPlatform/Dependency/Shutdown.py b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
new file mode 100644
index 0000000..ba77bbd
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
@@ -0,0 +1,19 @@
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def killOnosNodes( nodeIps ):
+ """
+ Kill all components of Onos on
+ given list of ips
+
+ Ex) nodeIps = ['10.0.0.1', '10.0.0.2']
+ """
+ killResult = main.TRUE
+
+ for node in nodeIps:
+ killResult = killResult and main.ONOSbench.onosDie( node )
+ if killResult == main.TRUE:
+ main.log.info( str(node) + ' was killed' )
+
+ return killResult
diff --git a/TestON/tests/FuncPlatform/Dependency/Startup.py b/TestON/tests/FuncPlatform/Dependency/Startup.py
index 3096eee..f3f90a9 100644
--- a/TestON/tests/FuncPlatform/Dependency/Startup.py
+++ b/TestON/tests/FuncPlatform/Dependency/Startup.py
@@ -4,11 +4,13 @@
Guidelines:
* Group sequential functionalities together
- * Methods should not prohibit cross platform execution
+ * Methods should not be dependent on platform
* Return main.TRUE on success or comprehensive error message
on failure (TBD)
+ * All methods should be consistent in expected behavior
"""
import time
+import json
def __init__( self ):
self.ip = '127.0.0.1'
@@ -46,6 +48,10 @@
* Force install ONOS package
* Start ONOS service
* Start ONOS cli
+
+ Also verifies that Onos is up and running by
+ 'isup' driver function which executs
+ 'onos-wait-for-start'
"""
# NOTE: leave out create cell file until bug addressed
@@ -69,8 +75,145 @@
for node in range( 0, numNodes ):
cli = cli and main.CLIs[node].startOnosCli( onosIps[node] )
- if sc and vc and op and oi and iu and cli == main.TRUE:
+ # Check if all nodes are discovered correctly using
+ # 'nodes' command in Onos Cli
+ na = main.TRUE
+ try:
+ nodeCmdJson = json.loads( main.CLIs[0].nodes() )
+ for node in nodeCmdJson:
+ if node['state'] != 'ACTIVE':
+ main.log.warn( str( node['id'] ) +
+ ' Node is not in ACTIVE state.' )
+ na = main.FALSE
+ if na != main.FALSE:
+ main.log.info( 'All nodes discovered successfully' )
+ except Exception:
+ main.log.error( 'nodes command did not execute properly' )
+ return main.FALSE
+
+ if sc and vc and op and oi and iu and cli and na == main.TRUE:
return main.TRUE
else:
return main.FALSE
+def installOnosFromTar( wgetAddr, nodeIps ):
+ """
+ Install Onos directly from tar.gz file.
+ Due to the nature of the specific steps required
+ to startup Onos in this fashion, all commands
+ required to start Onos from tar.gz will be
+ grouped in this method.
+
+ 1) wget latest onos tar.gz on onos node
+ 2) untar package
+ 3) specify onos-config cluster
+ 4) start onos via onos-service
+ 5) form onos cluster using onos-form-cluster
+ 6) check for successful startup
+
+ Specify the download link for the tar.gz.
+ Provide a list of nodeIps
+
+ Ex) wgetAddr = 'https://mytargzdownload.com/file.tar.gz'
+ nodeIps = ['10.0.0.1', '10.0.0.2']
+ """
+ if isinstance( nodeIps, ( int, basestring ) ):
+ main.log.error( 'Please pass in a list of string nodes' )
+ return main.FALSE
+
+ # Obtain filename from provided address
+ # assumes that filename is separated by '/' character
+ f_name = ''
+ addr = str( wgetAddr ).split('/')
+ for phrase in addr:
+ if 'tar.gz' in phrase:
+ f_name = str( phrase )
+ main.log.info( 'Using ' + f_name + ' as file' )
+
+ clusterCount = len( nodeIps )
+
+ main.log.info( 'Initiating Onos installation sequence ' +
+ 'using tar.gz ... This may take a few minutes' )
+
+ for node in range( 0, clusterCount ):
+ # Use the wgetAddr to download a new tar.gz from server
+ try:
+ main.ONOSnode[node].handle.sendline( 'wget ' + wgetAddr )
+ main.ONOSnode[node].handle.expect( 'saved' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ main.log.info( 'Successfully downloaded tar.gz ' +
+ 'on node: ' + str( main.ONOSips[node] ) )
+ except Exception:
+ # NOTE: Additional exception may be appropriate
+ main.log.error( 'Uncaught exception while ' +
+ 'downloading Onos tar.gz: ' +
+ main.ONOSnode[node].handle.before )
+ return main.FALSE
+
+ for node in range( 0, clusterCount ):
+ # Untar files on all nodes, then enter the
+ # newly created directory
+ try:
+ main.ONOSnode[node].handle.sendline( 'tar zxvf ' + f_name )
+ # Verbose output of tar will contain some onos returns
+ main.ONOSnode[node].handle.expect( 'onos' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ # NOTE: Making an assumption here:
+ # the directory created by tar file has a name that
+ # starts with 'onos-' followed by version number
+ # '1.2.0'. If this is NOT true, this method must
+ # be changed to enter the correct directory.
+ # The directory name is currently dynamic
+ # and depends on the day at which you downloaded
+ # the tar file
+ # Enter onos- wildcard to disregard its version / date
+ # suffix
+ main.ONOSnode[node].handle.sendline( 'cd onos-*' )
+ main.ONOSnode[node].handle.expect( '\$' )
+
+ main.ONOSnode[node].handle.sendline( 'pwd' )
+ main.ONOSnode[node].handle.expect( 'pwd' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ pwd = main.ONOSnode[node].handle.before
+ if 'onos' in str(pwd):
+ main.log.info( 'tar zxvf ' + f_name + ' successful ' +
+ 'on node ' + str( main.ONOSips[node] ) )
+
+ except Exception:
+ main.log.error( 'Uncaught exception while executing ' +
+ 'tar zxvf of ' + f_name + ': ' +
+ main.ONOSnode[node].handle.before +
+ main.ONOSnode[node].handle.after)
+ return main.FALSE
+
+ for node in range( 0, clusterCount ):
+ try:
+ main.ONOSnode[node].handle.sendline( 'bin/onos-service '+
+ 'server &' )
+ # Send some extra characters to run the process
+ main.ONOSnode[node].handle.sendline( '' )
+ main.ONOSnode[node].handle.sendline( '' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ except Exception:
+ main.log.error( 'Uncaught exception while executing ' +
+ 'onos-service server command ' +
+ str( main.ONOSnode[node].handle.before ) )
+ return main.FALSE
+
+ iu = main.TRUE
+ for node in nodeIps:
+ iu = iu and main.ONOSbench.isup( node )
+
+ if iu == main.TRUE:
+ return main.TRUE
+ else:
+ return main.FALSE
+
+def addAndStartOnosNode( nodeIps ):
+ """
+ A scale-out scenario that adds specified list of
+ nodes and starts those instances.
+
+ Ex) nodeIps = ['10.0.0.2', '10.0.0.3', 10.0.0.4']
+ """
+ main.log.info( 'addAndStartOnosNode implement me!' )
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.params b/TestON/tests/FuncPlatform/FuncPlatform.params
index eb274ae..3895660 100644
--- a/TestON/tests/FuncPlatform/FuncPlatform.params
+++ b/TestON/tests/FuncPlatform/FuncPlatform.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,3</testcases>
+ <testcases>1,2,3,4</testcases>
<DEP>
<startupSrc>
/home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Startup.py
@@ -14,6 +14,24 @@
<appClassName>
App
</appClassName>
+
+ <logSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Logger.py
+ </logSrc>
+ <logClassName>
+ Log
+ </logClassName>
+
+ <shutdownSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Shutdown.py
+ </shutdownSrc>
+ <shutdownClassName>
+ Shutdown
+ </shutdownClassName>
+
+ <targz>
+ http://downloads.onosproject.org/nightly/onos-1.2.0.latest-NIGHTLY.tar.gz
+ </targz>
</DEP>
<CTRL>
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.py b/TestON/tests/FuncPlatform/FuncPlatform.py
index 6bcb06c..60b1aa0 100644
--- a/TestON/tests/FuncPlatform/FuncPlatform.py
+++ b/TestON/tests/FuncPlatform/FuncPlatform.py
@@ -19,6 +19,10 @@
3. Activate application Y
4. Deactivate application X
+The ideal platform test script should have incredible
+robustness to possible exceptions and report the most
+useful error messages.
+
contributers to contact for help:
andrew@onlab.us
"""
@@ -30,7 +34,13 @@
def CASE1( self, main ):
"""
Main scope initialization case
+ Must include to run any other test cases
"""
+ import imp
+
+ # NOTE: Hardcoded application name subject to change
+ # closely monitor and make changes when necessary
+ # (or implement ways to dynamically get names)
main.appList = {
'bgprouter' : 'org.onosproject.bgprouter',
'config' : 'org.onosproject.config',
@@ -46,23 +56,59 @@
'mobility' : 'org.onosproject.mobility',
'netconf' : 'org.onosproject.netconf',
'null' : 'org.onosproject.null',
- 'optical' : 'org.onosproject.optical'
+ 'optical' : 'org.onosproject.optical',
+ 'pcep' : 'org.onosproject.pcep',
+ 'proxyarp' : 'org.onosproject.proxyarp',
+ 'reactive.routing' : 'org.onosproject.reactive.routing',
+ 'sdnip' : 'org.onosproject.sdnip',
+ 'segmentrouting' : 'org.onosproject.segmentrouting',
+ 'tunnel' : 'org.onosproject.tunnel',
+ 'virtualbng' : 'org.onosproject.virtualbng',
+ 'xosintegration' : 'org.onosproject.xosintegration'
}
# List of ONOS ip's specififed in params
main.ONOSips = []
main.CLIs = []
+ main.ONOSnode = []
for node in range( 0, int(main.params['CTRL']['num']) ):
main.ONOSips.append( main.params['CTRL']['ip'+str(node+1)] )
main.CLIs.append(
getattr( main, 'ONOS' + str(node+1) + 'cli' ) )
+ main.ONOSnode.append(
+ getattr( main, 'ONOS' + str(node+1) ) )
- def CASE2( self, main ):
- import time
- import imp
-
+ # Application source and name definitions
startupSrc = main.params['DEP']['startupSrc']
startupClassName = main.params['DEP']['startupClassName']
+
+ appClassName = main.params['DEP']['appClassName']
+ appSrc = main.params['DEP']['appSrc']
+
+ logClassName = main.params['DEP']['logClassName']
+ logSrc = main.params['DEP']['logSrc']
+
+ shutdownClassName = main.params['DEP']['shutdownClassName']
+ shutdownSrc = main.params['DEP']['shutdownSrc']
+
+ # Importing dependency class(es)
+ # Refer to source files in Dependency folder to
+ # make changes to its respective methods
+ # Be weary of naming collisions
+ try:
+ main.startup = imp.load_source( startupClassName, startupSrc )
+ main.app = imp.load_source( appClassName, appSrc )
+ main.onosLog = imp.load_source( logClassName, logSrc )
+ main.shutdown = imp.load_source( shutdownClassName, shutdownSrc )
+ except ImportError:
+ main.log.error( 'Error importing class file(s). Please ' +
+ 'check file location' )
+ main.cleanup()
+ main.exit()
+
+ def CASE2( self, main ):
+ import time
+
cellName = main.params['CELL']['name']
appStr = main.params['CELL']['appStr']
benchIp = main.params['BENCH']['ip']
@@ -70,21 +116,10 @@
gitPull = main.params['GIT']['pull']
mnIp = main.params['MN']['ip']
- # importing dependency class(es)
- # Refer to source files in Dependency folder to
- # make changes to its respective methods
- try:
- startup = imp.load_source( startupClassName, startupSrc )
- except ImportError:
- main.log.error( "Error importing class " +
- str(startupClassName) + " from " + str(startupSrc) )
- main.cleanup()
- main.exit()
-
main.case( 'Setup environment and install ONOS' )
if gitPull == 'on':
main.step( 'Git pull and clean install' )
- gitPullResult = startup.gitPullAndMci( branchName )
+ gitPullResult = main.startup.gitPullAndMci( branchName )
utilities.assert_equals( expect=main.TRUE,
actual=gitPullResult,
onpass='Git pull and install successful',
@@ -92,7 +127,7 @@
str(gitPullResult) )
main.step( 'Initiate ONOS startup sequence' )
- startupResult = startup.initOnosStartupSequence(
+ startupResult = main.startup.initOnosStartupSequence(
cellName, appStr, benchIp, mnIp, main.ONOSips )
utilities.assert_equals( expect=main.TRUE,
actual=startupResult,
@@ -102,31 +137,58 @@
def CASE3( self, main ):
import time
- import imp
main.case( 'Activate applications and check installation' )
- # Activate applications and check consistency
- # across clusters
- appClassName = main.params['DEP']['appClassName']
- appSrc = main.params['DEP']['appSrc']
+
+ # NOTE: Test only
+ # Unceremoniously kill onos 2
+ main.ONOSbench.onosDie( '10.128.174.2' )
+
+ time.sleep( 30 )
- # Import application file to use its methods
- try:
- app = imp.load_source( appClassName, appSrc )
- except ImportError:
- main.log.error( "Error importing class " +
- str(startupClassName) + " from " + str(startupSrc) )
- main.cleanup()
- main.exit()
+ main.step( 'Sample Onos log check' )
+ logResult = main.onosLog.checkOnosLog( main.ONOSips[0] )
+ main.log.info( logResult )
+ # TODO: Define pass criteria
+ utilities.assert_equals( expect=main.TRUE,
+ actual=main.TRUE,
+ onpass= 'Logging successful',
+ onfail= 'Logging failed ' )
# Sample app activation
main.step( 'Activating applications metrics and fwd' )
appList = ['metrics', 'fwd']
- appResult = app.activate( appList )
+ appResult = main.app.activate( appList )
utilities.assert_equals( expect=main.TRUE,
actual=appResult,
onpass= 'App activation of ' + str(appList) + ' successful',
onfail= 'App activation failed ' + str(appResult) )
+ def CASE4( self, main ):
+ """
+ Download ONOS tar.gz built from latest nightly
+ (following tutorial on wiki) and run ONOS directly on the
+ instance
+ """
+ import imp
+
+ targz = main.params['DEP']['targz']
+ clusterCount = main.params['CTRL']['num']
+
+ main.case( 'Install ONOS from onos.tar.gz file' )
+
+ main.step( 'Killing all ONOS instances previous started' )
+ killResult = main.shutdown.killOnosNodes( main.ONOSips )
+ utilities.assert_equals( expect=main.TRUE,
+ actual = killResult,
+ onpass = 'All Onos nodes successfully killed',
+ onfail = 'Onos nodes were not successfully killed' )
+
+ main.step( 'Starting ONOS using tar.gz on all nodes' )
+ installResult = main.startup.installOnosFromTar( targz, main.ONOSips )
+ utilities.assert_equals( expect=main.TRUE,
+ actual = installResult,
+ onpass= 'Onos tar.gz installation successful',
+ onfail= 'Onos tar.gz installation failed' )
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.topo b/TestON/tests/FuncPlatform/FuncPlatform.topo
index fcc087d..f19e8f8 100644
--- a/TestON/tests/FuncPlatform/FuncPlatform.topo
+++ b/TestON/tests/FuncPlatform/FuncPlatform.topo
@@ -14,7 +14,7 @@
<host>10.128.174.1</host>
<user>admin</user>
<password></password>
- <type>OnosCliDriver</type>
+ <type>OnosDriver</type>
<connect_order>3</connect_order>
<COMPONENTS> </COMPONENTS>
</ONOS1>
@@ -23,7 +23,7 @@
<host>10.128.174.2</host>
<user>admin</user>
<password></password>
- <type>OnosCliDriver</type>
+ <type>OnosDriver</type>
<connect_order>4</connect_order>
<COMPONENTS> </COMPONENTS>
</ONOS2>
@@ -32,7 +32,7 @@
<host>10.128.174.3</host>
<user>admin</user>
<password></password>
- <type>OnosCliDriver</type>
+ <type>OnosDriver</type>
<connect_order>5</connect_order>
<COMPONENTS> </COMPONENTS>
</ONOS3>