Allow use of onos docker for existing tests
- Allow Cluster to pull/build onos docker
- Connect clidriver to cli runnning in docker
- Some changes for debugability in components
- To use, set the useDocker and diffCliHosts tags in the cluster
component to True, then define parameters in the params file
- Update all SR Stratum tests to use the tost docker image
- NOTE: Since the tost-onos image doesn't have openflow installe, we are
currently only using docker for the bmv2 and tofino switches
Change-Id: If900b0bdbf9a41b8885c692ccba18a3b1bc580cc
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
index 9ed9256..ae08de9 100644
--- a/TestON/tests/dependencies/Cluster.py
+++ b/TestON/tests/dependencies/Cluster.py
@@ -37,7 +37,7 @@
atomixNodes.append( "{%s:%s}" % ( node.name, node.ipAddress ) )
return "%s[%s; Atomix Nodes:%s]" % ( self.name, ", ".join( controllers ), ", ".join( atomixNodes ) )
- def __init__( self, ctrlList=[], name="Cluster" ):
+ def __init__( self, ctrlList=[], name="Cluster", useDocker=False ):
"""
controllers : All the nodes
runningNodes : Node that are specifically running from the test.
@@ -53,6 +53,16 @@
self.name = str( name )
self.atomixNodes = ctrlList
self.iterator = iter( self.active() )
+ self.useDocker = useDocker
+ clusterParams = main.params.get( "CLUSTER", {} )
+ self.dockerSkipBuild = clusterParams.get( "dockerSkipBuild", False )
+ self.dockerBuildCmd = clusterParams.get( "dockerBuildCmd", None )
+ self.dockerBuildTimeout = int( clusterParams.get( "dockerBuildTimeout", 600 ) )
+ self.dockerFilePath = clusterParams.get( "dockerFilePath", None )
+ self.dockerImageTag = clusterParams.get( "dockerImageTag", None )
+ self.dockerOptions = clusterParams.get( "dockerOptions", "" )
+ self.atomixImageTag = clusterParams.get( "atomixImageTag", None )
+ self.atomixOptions = clusterParams.get( "atomixOptions", "" )
def fromNode( self, ctrlList ):
"""
@@ -389,6 +399,88 @@
ctrlList[ i ].active = False
return result
+ def dockerStop( self, killMax, atomix=True ):
+ """
+ Description:
+ killing the onos docker containers. It will either kill the
+ current runningnodes or max number of the nodes.
+ Required:
+ * killRemoveMax - The boolean that will decide either to kill
+ only running nodes ( False ) or max number of nodes ( True ).
+ Returns:
+ Returns main.TRUE if successfully killing it.
+ """
+ getFrom = "all" if killMax else "running"
+ result = main.TRUE
+ stopResult = self.command( "dockerStop",
+ args=[ "name" ],
+ specificDriver=4,
+ getFrom=getFrom,
+ funcFromCtrl=True )
+ ctrlList = self.fromNode( getFrom )
+ for i in range( len( stopResult ) ):
+ result = result and stopResult[ i ]
+ ctrlList[ i ].active = False
+ atomixResult = main.TRUE
+ if atomix:
+ atomixResult = self.stopAtomixDocker( killMax )
+ return result and atomixResult
+
+ def dockerBuild( self, pull=True ):
+ """
+ Description:
+ Build ONOS docker image
+ Optional:
+ * pull - Try to pull latest image before building
+ Returns:
+ Returns main.TRUE if successfully killing it.
+ """
+ getFrom = "all"
+ result = main.TRUE
+ atomixResult = []
+ buildResult = []
+ if self.atomixImageTag:
+ atomixResult = self.command( "dockerPull",
+ args=[ self.atomixImageTag ],
+ specificDriver=4,
+ getFrom=getFrom,
+ funcFromCtrl=False )
+ if not self.dockerImageTag:
+ main.log.error( "No image given, exiting test" )
+ return main.FALSE
+ if pull and self.dockerImageTag:
+ buildResult = self.command( "dockerPull",
+ args=[ self.dockerImageTag ],
+ specificDriver=4,
+ getFrom=getFrom,
+ funcFromCtrl=False )
+ for i in range( len( buildResult ) ):
+ result = result and buildResult[ i ]
+ if self.dockerSkipBuild:
+ return main.TRUE
+ if not result and self.dockerBuildCmd:
+ buildResult = self.command( "makeDocker",
+ args=[ self.dockerFilePath, self.dockerBuildCmd ],
+ kwargs={ "timeout": self.dockerBuildTimeout,
+ "prompt": "Successfully tagged %s" % self.dockerImageTag },
+ specificDriver=4,
+ getFrom=getFrom,
+ funcFromCtrl=False )
+
+ elif not result:
+ buildResult = self.command( "dockerBuild",
+ args=[ self.dockerFilePath, self.dockerImageTag ],
+ kwargs={ "timeout": self.dockerBuildTimeout,
+ "pull": pull },
+ specificDriver=4,
+ getFrom=getFrom,
+ funcFromCtrl=False )
+ for i in range( len( atomixResult ) ):
+ result = result and atomixResult[ i ]
+ for i in range( len( buildResult ) ):
+ result = result and buildResult[ i ]
+ return result
+
def ssh( self ):
"""
Description:
@@ -399,9 +491,16 @@
the onos.
"""
result = main.TRUE
+ if self.useDocker:
+ driver = 2
+ kwargs = { "userName": "karafUser",
+ "userPWD": "karafPass" }
+ else:
+ driver = 1
+ kwargs = { "node": "ipAddress" }
sshResult = self.command( "onosSecureSSH",
- kwargs={ "node": "ipAddress" },
- specificDriver=1,
+ kwargs=kwargs,
+ specificDriver=driver,
getFrom="running",
funcFromCtrl=True )
for sshR in sshResult:
@@ -417,6 +516,9 @@
Returns main.TRUE if it successfully installed
"""
result = main.TRUE
+ if self.useDocker:
+ # We will do this as part of startDocker
+ return result
threads = []
i = 0
for ctrl in self.atomixNodes:
@@ -472,6 +574,124 @@
result = result and t.result
return result
+ def startONOSDocker( self, installMax=True, installParallel=True ):
+ """
+ Description:
+ Installing onos via docker containers.
+ Required:
+ * installMax - True for installing max number of nodes
+ False for installing current running nodes only.
+ Returns:
+ Returns main.TRUE if it successfully installed
+ """
+ result = main.TRUE
+ threads = []
+ for ctrl in self.controllers if installMax else self.runningNodes:
+ if installParallel:
+ t = main.Thread( target=ctrl.server.dockerRun,
+ name="onos-run-docker-" + ctrl.name,
+ args=[ self.dockerImageTag, ctrl.name ],
+ kwargs={ "options" : self.dockerOptions } )
+ threads.append( t )
+ t.start()
+ else:
+ result = result and \
+ ctrl.server.dockerRun( self.dockerImageTag,
+ ctrl.name,
+ options=self.dockerOptions )
+ if installParallel:
+ for t in threads:
+ t.join()
+ result = result and t.result
+ return result
+
+ def startAtomixDocker( self, installParallel=True ):
+ """
+ Description:
+ Installing atomix via docker containers.
+ Required:
+ * installParallel - True for installing atomix in parallel.
+ Returns:
+ Returns main.TRUE if it successfully installed
+ """
+ result = main.TRUE
+ threads = []
+ for ctrl in self.atomixNodes:
+ if installParallel:
+ t = main.Thread( target=ctrl.server.dockerRun,
+ name="atomix-run-docker-" + ctrl.name,
+ args=[ self.atomixImageTag, "atomix-" + ctrl.name ],
+ kwargs={ "options" : main.params['CLUSTER']['atomixOptions'],
+ "imageArgs": " --config /opt/atomix/conf/atomix.json --ignore-resources"} )
+ threads.append( t )
+ t.start()
+ else:
+ result = result and \
+ ctrl.server.dockerRun( self.atomixImageTag,
+ "atomix-" + ctrl.name,
+ options=main.params['CLUSTER']['atomixOptions'] )
+ if installParallel:
+ for t in threads:
+ t.join()
+ result = result and t.result
+ return result
+
+ def stopAtomixDocker( self, killMax=True, installParallel=True ):
+ """
+ Description:
+ Stoping all atomix containers
+ Required:
+ * killMax - True for stoping max number of nodes
+ False for stoping current running nodes only.
+ Returns:
+ Returns main.TRUE if it successfully stoped
+ """
+ result = main.TRUE
+ threads = []
+ for ctrl in self.controllers if killMax else self.atomixNodes:
+ if installParallel:
+ t = main.Thread( target=ctrl.server.dockerStop,
+ name="atomix-stop-docker-" + ctrl.name,
+ args=[ "atomix-" + ctrl.name ] )
+ threads.append( t )
+ t.start()
+ else:
+ result = result and \
+ ctrl.server.dockerStop( "atomix-" + ctrl.name )
+ if installParallel:
+ for t in threads:
+ t.join()
+ result = result and t.result
+ return result
+
+ def genPartitions( self, path="/tmp/cluster.json" ):
+ """
+ Description:
+ Create cluster config and move to each onos server
+ Required:
+ * installMax - True for installing max number of nodes
+ False for installing current running nodes only.
+ Returns:
+ Returns main.TRUE if it successfully installed
+ """
+ result = main.TRUE
+ # move files to onos servers
+ for ctrl in self.atomixNodes:
+ localAtomixFile = ctrl.ip_address + "-atomix.json"
+ result = result and main.ONOSbench.generateAtomixConfig( ctrl.server.ip_address, path=localAtomixFile )
+ result = result and main.ONOSbench.scp( ctrl.server,
+ localAtomixFile,
+ "/tmp/atomix.json",
+ direction="to" )
+ for ctrl in self.controllers:
+ localOnosFile = ctrl.ip_address + "-cluster.json"
+ result = result and main.ONOSbench.generateOnosConfig( ctrl.server.ip_address, path=localOnosFile )
+ result = result and main.ONOSbench.scp( ctrl.server,
+ localOnosFile,
+ path,
+ direction="to" )
+ return result
+
def startCLIs( self ):
"""
Description:
@@ -522,6 +742,16 @@
main.log.warn( repr( i ) )
currentResult = False
results = results and currentResult
+ # Check to make sure all bundles are started
+ bundleOutput = self.command( "sendline", args=[ "bundle:list" ] )
+ for i in bundleOutput:
+ if "START LEVEL 100" in i:
+ currentResult = True
+ else:
+ currentResult = False
+ main.log.warn( "Node's bundles not fully started" )
+ main.log.debug( i )
+ results = results and currentResult
return results
def appsCheck( self, apps ):
@@ -548,6 +778,50 @@
main.log.warn( "{}: {} is in {} state".format( ctrl.name, app, states[ i ] ) )
return results
+ def attachToONOSDocker( self ):
+ """
+ Description:
+ connect to onos docker using onosCli driver
+ Required:
+ Returns:
+ Returns main.TRUE if it successfully started.
+ """
+ getFrom = "running"
+ result = main.TRUE
+ execResults = self.command( "dockerExec",
+ args=[ "name" ],
+ kwargs={ "dockerPrompt": "dockerPrompt" },
+ specificDriver=2,
+ getFrom=getFrom,
+ funcFromCtrl=True )
+ ctrlList = self.fromNode( getFrom )
+ for i in range( len( execResults ) ):
+ result = result and execResults[ i ]
+ ctrlList[ i ].active = True
+ return result
+
+ def prepareForCLI( self ):
+ """
+ Description:
+ prepare docker to connect to the onos cli
+ Required:
+ Returns:
+ Returns main.TRUE if it successfully started.
+ """
+ getFrom = "running"
+ for ctrl in self.getRunningNodes():
+ ctrl.CLI.inDocker = True
+ result = main.TRUE
+ execResults = self.command( "prepareForCLI",
+ specificDriver=2,
+ getFrom=getFrom,
+ funcFromCtrl=True )
+ ctrlList = self.fromNode( getFrom )
+ for i in range( len( execResults ) ):
+ result = result and execResults[ i ]
+ ctrlList[ i ].active = True
+ return result
+
def printResult( self, results, activeList, logLevel="debug" ):
"""
Description:
@@ -623,6 +897,7 @@
1 - from bench
2 - from cli
3 - from rest
+ 4 - from server
* contentCheck - If this is True, it will check if the result has some
contents.
* getFrom - from which nodes
@@ -637,24 +912,34 @@
Returns resultContent of the result if contentCheck
"""
threads = []
- drivers = [ None, "Bench", "CLI", "REST" ]
+ drivers = [ None, "Bench", "CLI", "REST", "server" ]
results = []
for ctrl in self.fromNode( getFrom ):
+ funcArgs = []
+ funcKwargs = {}
try:
- funcArgs = []
- funcKwargs = {}
f = getattr( ( ctrl if not specificDriver else
getattr( ctrl, drivers[ specificDriver ] ) ), function )
- if funcFromCtrl:
- if args:
- for i in range( len( args ) ):
- funcArgs.append( getattr( ctrl, args[ i ] ) )
- if kwargs:
- for k in kwargs:
- funcKwargs.update( { k: getattr( ctrl, kwargs[ k ] ) } )
except AttributeError:
main.log.error( "Function " + function + " not found. Exiting the Test." )
main.cleanAndExit()
+ if funcFromCtrl:
+ if args:
+ try:
+ for i in range( len( args ) ):
+ funcArgs.append( getattr( ctrl, args[ i ] ) )
+ except AttributeError:
+ main.log.error( "Argument " + str( args[ i ] ) + " for " + str( f ) + " not found. Exiting the Test." )
+ main.cleanAndExit()
+ if kwargs:
+ try:
+ for k in kwargs:
+ funcKwargs.update( { k: getattr( ctrl, kwargs[ k ] ) } )
+ except AttributeError as e:
+ main.log.exception("")
+ main.log.error( "Keyword Argument " + str( k ) + " for " + str( f ) + " not found. Exiting the Test." )
+ main.log.debug( "Passed kwargs: %s; dir(ctrl): %s" % ( repr( kwargs ), dir( ctrl ) ) )
+ main.cleanAndExit()
t = main.Thread( target=f,
name=function + "-" + ctrl.name,
args=funcArgs if funcFromCtrl else args,
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index fabfe9a..020af9d 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -44,7 +44,7 @@
try:
main.Cluster
except ( NameError, AttributeError ):
- main.Cluster = Cluster( main.ONOScell.nodes )
+ main.Cluster = Cluster( main.ONOScell.nodes, useDocker=main.ONOScell.useDocker )
main.ONOSbench = main.Cluster.controllers[ 0 ].Bench
main.testOnDirectory = re.sub( "(/tests)$", "", main.testsRoot )
@@ -100,20 +100,20 @@
try:
main.Cluster
except ( NameError, AttributeError ):
- main.Cluster = Cluster( main.ONOScell.nodes )
+ main.Cluster = Cluster( main.ONOScell.nodes, useDocker=main.ONOScell.useDocker )
main.cellData = {} # For creating cell file
return main.TRUE
- def envSetupException( self, e ):
+ def envSetupException( self, error ):
"""
Description:
handles the exception that might occur from the environment setup.
Required:
- * includeGitPull - exceeption code e.
+ * error - exception returned from except.
"""
- main.log.exception( e )
+ main.log.exception( error )
main.cleanAndExit()
def envSetupConclusion( self, stepResult ):
@@ -219,6 +219,21 @@
main.log.info( "Safety check, killing all ONOS processes" )
return cluster.killOnos( killRemoveMax, stopOnos )
+ def killingAllOnosDocker( self, cluster, killRemoveMax ):
+ """
+ Description:
+ killing the onos docker images . It will either kill the
+ current runningnodes or max number of the nodes.
+ Required:
+ * cluster - the cluster driver that will be used.
+ * killRemoveMax - The boolean that will decide either to kill
+ only running nodes ( False ) or max number of nodes ( True ).
+ Returns:
+ Returns main.TRUE if successfully killing it.
+ """
+ main.log.info( "Safety check, stopping all ONOS docker containers" )
+ return cluster.dockerStop( killRemoveMax )
+
def createApplyCell( self, cluster, newCell, cellName, cellApps,
mininetIp, useSSH, onosIps, installMax=False,
atomixClusterSize=None ):
@@ -243,6 +258,8 @@
"""
if atomixClusterSize is None:
atomixClusterSize = len( cluster.runningNodes )
+ if atomixClusterSize is 1:
+ atomixClusterSize = len( cluster.controllers )
atomixClusterSize = int( atomixClusterSize )
cluster.setAtomixNodes( atomixClusterSize )
atomixIps = [ node.ipAddress for node in cluster.atomixNodes ]
@@ -316,6 +333,25 @@
main.cleanAndExit()
return packageResult
+ def buildDocker( self, cluster ):
+ """
+ Description:
+ Build the latest docker
+ Required:
+ * cluster - the cluster driver that will be used.
+ Returns:
+ Returns main.TRUE if it successfully built.
+ """
+ main.step( "Building ONOS Docker image" )
+ buildResult = cluster.dockerBuild()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=buildResult,
+ onpass="Successfully created ONOS docker",
+ onfail="Failed to create ONOS docker" )
+ if not buildResult:
+ main.cleanAndExit()
+ return buildResult
+
def installAtomix( self, cluster, parallel=True ):
"""
Description:
@@ -361,6 +397,42 @@
main.cleanAndExit()
return onosInstallResult
+ def startDocker( self, cluster, installMax, parallel=True ):
+ """
+ Description:
+ Start onos docker containers and verify the result
+ Required:
+ * cluster - the cluster driver that will be used.
+ * installMax - True for installing max number of nodes
+ False for installing current running nodes only.
+ Returns:
+ Returns main.TRUE if it successfully installed
+ """
+ main.step( "Create Cluster Config" )
+ configResult = cluster.genPartitions()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=configResult,
+ onpass="Successfully create cluster config",
+ onfail="Failed to create cluster config" )
+
+ # install atomix docker containers
+ main.step( "Installing Atomix via docker containers" )
+ atomixInstallResult = cluster.startAtomixDocker( parallel )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=atomixInstallResult,
+ onpass="Successfully start atomix containers",
+ onfail="Failed to start atomix containers" )
+
+ main.step( "Installing ONOS via docker containers" )
+ onosInstallResult = cluster.startONOSDocker( installMax, parallel )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=onosInstallResult,
+ onpass="Successfully start ONOS containers",
+ onfail="Failed to start ONOS containers" )
+ if not onosInstallResult and atomixInstallResult:
+ main.cleanAndExit()
+ return onosInstallResult and atomixInstallResult
+
def setupSsh( self, cluster ):
"""
Description:
@@ -565,6 +637,7 @@
if restartCluster:
atomixKillResult = self.killingAllAtomix( cluster, killRemoveMax, stopAtomix )
onosKillResult = self.killingAllOnos( cluster, killRemoveMax, stopOnos )
+ dockerKillResult = self.killingAllOnosDocker( cluster, killRemoveMax )
killResult = atomixKillResult and onosKillResult
else:
killResult = main.TRUE
@@ -577,11 +650,24 @@
packageResult = main.TRUE
if not skipPack:
- packageResult = self.buildOnos(cluster)
+ if cluster.useDocker:
+ packageResult = self.buildDocker( cluster )
+ else:
+ packageResult = self.buildOnos( cluster )
- atomixInstallResult = self.installAtomix( cluster, installParallel )
- onosInstallResult = self.installOnos( cluster, installMax, installParallel )
- installResult = atomixInstallResult and onosInstallResult
+ if cluster.useDocker:
+ installResult = self.startDocker( cluster, installMax, installParallel )
+ else:
+ atomixInstallResult = self.installAtomix( cluster, installParallel )
+ onosInstallResult = self.installOnos( cluster, installMax, installParallel )
+ installResult = atomixInstallResult and onosInstallResult
+
+ preCLIResult = main.TRUE
+ if cluster.useDocker:
+ attachResult = cluster.attachToONOSDocker()
+ prepareResult = cluster.prepareForCLI()
+
+ preCLIResult = preCLIResult and attachResult and prepareResult
self.processList( extraClean, cleanArgs )
secureSshResult = self.setupSsh( cluster )
@@ -590,8 +676,11 @@
uninstallResult = main.TRUE
installResult = main.TRUE
secureSshResult = main.TRUE
+ preCLIResult = main.TRUE
- onosServiceResult = self.checkOnosService( cluster )
+ onosServiceResult = main.TRUE
+ if not cluster.useDocker:
+ onosServiceResult = self.checkOnosService( cluster )
onosCliResult = main.TRUE
if startOnosCli:
@@ -604,6 +693,11 @@
if apps:
apps = apps.split( ',' )
apps = [ appPrefix + app for app in apps ]
+ if cluster.useDocker:
+ node = main.Cluster.active( 0 )
+ for app in apps:
+ node.activateApp( app )
+
onosAppsResult = self.checkOnosApps( cluster, apps )
else:
main.log.warn( "No apps were specified to be checked after startup" )
@@ -616,4 +710,4 @@
return killResult and cellResult and packageResult and uninstallResult and \
installResult and secureSshResult and onosServiceResult and onosCliResult and \
- onosNodesResult and onosAppsResult
+ onosNodesResult and onosAppsResult and preCLIResult
diff --git a/TestON/tests/dependencies/utils.py b/TestON/tests/dependencies/utils.py
index 6537afc..3cf849a 100644
--- a/TestON/tests/dependencies/utils.py
+++ b/TestON/tests/dependencies/utils.py
@@ -74,10 +74,20 @@
scpResult = main.TRUE
copyResult = main.TRUE
for ctrl in main.Cluster.runningNodes:
- scpResult = scpResult and main.ONOSbench.scp( ctrl,
- "/opt/onos/log/karaf.log",
- "/tmp/karaf.log",
- direction="from" )
+ if ctrl.inDocker:
+ scpResult = scpResult and ctrl.server.dockerCp( ctrl.name,
+ "/opt/onos/log/karaf.log",
+ "/tmp/karaf.log",
+ direction="from" )
+ scpResult = scpResult and main.ONOSbench.scp( ctrl.server,
+ "/tmp/karaf.log",
+ "/tmp/karaf.log",
+ direction="from" )
+ else:
+ scpResult = scpResult and main.ONOSbench.scp( ctrl,
+ "/opt/onos/log/karaf.log",
+ "/tmp/karaf.log",
+ direction="from" )
copyResult = copyResult and main.ONOSbench.cpLogsToDir( "/tmp/karaf.log", main.logdir,
copyFileName=( copyFileName + "_karaf.log." +
ctrl.name + "_" ) if before else