[ONOS-7756] Start/stop Atomix cluster as part of ONOS cluster setup
Change-Id: Ib2af5e14af9cc59ae9d9cc90b54a91db4914a3a4
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 89dc363..8890871 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -2668,3 +2668,111 @@
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+
+ def atomixKill( self, nodeIp ):
+ """
+ Calls the command: 'atomix-kill [<node-ip>]'
+ Kills the Atomix instance running on the specified node
+ """
+ try:
+ self.handle.sendline( "" )
+ self.handle.expect( self.prompt )
+ self.handle.sendline( "atomix-kill " + str( nodeIp ) )
+ i = self.handle.expect( [
+ self.prompt,
+ "No\sroute\sto\shost",
+ "password:",
+ pexpect.TIMEOUT ], timeout=60 )
+
+ if i == 0:
+ main.log.info( "Atomix instance " + str( nodeIp ) + " was killed" )
+ return main.TRUE
+ elif i == 1:
+ main.log.info( "No route to host" )
+ return main.FALSE
+ elif i == 2:
+ main.log.info( "Passwordless login for host: " + str( nodeIp ) + " not configured" )
+ return main.FALSE
+ else:
+ main.log.info( "Atomix instance was not killed" )
+ return main.FALSE
+
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
+ def atomixUninstall( self, nodeIp="" ):
+ """
+ Calls the command: 'atomix-uninstall'
+ Uninstalls Atomix from the designated node, stopping if needed
+ """
+ try:
+ self.handle.sendline( "" )
+ self.handle.expect( self.prompt, timeout=180 )
+ self.handle.sendline( "atomix-uninstall " + str( nodeIp ) )
+ self.handle.expect( self.prompt, timeout=180 )
+ main.log.info( "Atomix " + nodeIp + " was uninstalled" )
+ # onos-uninstall command does not return any text
+ return main.TRUE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": Timeout in atomixUninstall" )
+ self.handle.send( "\x03" ) # Control-C
+ self.handle.expect( self.prompt )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
+ def atomixInstall( self, options="", node="" ):
+ """
+ Installs Atomix on the designated nodes.
+ Returns: main.TRUE on success and main.FALSE on failure
+ """
+ try:
+ if options:
+ self.handle.sendline( "atomix-install " + options + " " + node )
+ else:
+ self.handle.sendline( "atomix-install " + node )
+ self.handle.expect( "atomix-install " )
+ i = self.handle.expect( [ "Network\sis\sunreachable",
+ "is already installed",
+ "saved",
+ self.prompt,
+ pexpect.TIMEOUT ], timeout=180 )
+ if i == 0:
+ # can't reach ONOS node
+ main.log.warn( "Network is unreachable" )
+ self.handle.expect( self.prompt )
+ return main.FALSE
+ elif i == 1:
+ # same bits are already on Atomix node
+ main.log.info( "Atomix is already installed on " + node )
+ self.handle.expect( self.prompt )
+ return main.TRUE
+ elif i == 2 or i == 3:
+ main.log.info( "Atomix was installed on " + node )
+ self.handle.expect( self.prompt )
+ return main.TRUE
+ elif i == 4:
+ # timeout
+ main.log.info( "Installation of Atomix on " + node + " timed out" )
+ self.handle.expect( self.prompt )
+ main.log.warn( self.handle.before )
+ self.handle.send( "\x03" ) # Control-C
+ self.handle.expect( self.prompt )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
index a67bad5..6bfbb3f 100644
--- a/TestON/tests/dependencies/Cluster.py
+++ b/TestON/tests/dependencies/Cluster.py
@@ -176,7 +176,27 @@
specificDriver=1,
getFrom=0 if installMax else 1 )
- def uninstall( self, uninstallMax ):
+ def uninstallAtomix( self, uninstallMax ):
+ """
+ Description:
+ uninstalling atomix
+ Required:
+ * uninstallMax - True for uninstalling max number of nodes
+ False for uninstalling the current running nodes.
+ Returns:
+ Returns main.TRUE if it successfully uninstalled.
+ """
+ result = main.TRUE
+ uninstallResult = self.command( "atomixUninstall",
+ kwargs={ "nodeIp": "ipAddress" },
+ specificDriver=1,
+ getFrom=0 if uninstallMax else 1,
+ funcFromCtrl=True )
+ for uninstallR in uninstallResult:
+ result = result and uninstallR
+ return result
+
+ def uninstallOnos( self, uninstallMax ):
"""
Description:
uninstalling onos
@@ -244,7 +264,31 @@
main.log.warn( ctrl.name + " may not be up." )
return onosIsUp
- def kill( self, killMax, stopOnos ):
+ def killAtomix( self, killMax, stopAtomix ):
+ """
+ Description:
+ killing atomix. It will either kill the current runningnodes or
+ max number of the nodes.
+ Required:
+ * killRemoveMax - The boolean that will decide either to kill
+ only running nodes ( False ) or max number of nodes ( True ).
+ * stopAtomix - If wish to atomix onos before killing it. True for
+ enable stop, False for disable stop.
+ Returns:
+ Returns main.TRUE if successfully killing it.
+ """
+ result = main.TRUE
+ killResult = self.command( "atomixKill",
+ args=[ "ipAddress" ],
+ specificDriver=1,
+ getFrom=0 if killMax else 1,
+ funcFromCtrl=True )
+ for i in range( len( killResult ) ):
+ result = result and killResult[ i ]
+ self.controllers[ i ].active = False
+ return result
+
+ def killOnos( self, killMax, stopOnos ):
"""
Description:
killing the onos. It will either kill the current runningnodes or
@@ -287,7 +331,42 @@
result = result and sshR
return result
- def install( self, installMax=True, installParallel=True ):
+ def installAtomix( self, installMax=True, installParallel=True ):
+ """
+ Description:
+ Installing onos.
+ Required:
+ * installMax - True for installing max number of nodes
+ False for installing current running nodes only.
+ Returns:
+ Returns main.TRUE if it successfully installed
+ """
+ result = main.TRUE
+ threads = []
+ i = 0
+ for ctrl in self.controllers if installMax else self.runningNodes:
+ options = ""
+ if installMax and i >= self.numCtrls:
+ # TODO: is installMax supported here?
+ pass
+ if installParallel:
+ t = main.Thread( target=ctrl.Bench.atomixInstall,
+ name="atomix-install-" + ctrl.name,
+ kwargs={ "node" : ctrl.ipAddress,
+ "options" : options } )
+ threads.append( t )
+ t.start()
+ else:
+ result = result and \
+ main.ONOSbench.atomixInstall( node=ctrl.ipAddress, options=options )
+ i += 1
+ if installParallel:
+ for t in threads:
+ t.join()
+ result = result and t.result
+ return result
+
+ def installOnos( self, installMax=True, installParallel=True ):
"""
Description:
Installing onos.
@@ -306,7 +385,7 @@
options = "-nf"
if installParallel:
t = main.Thread( target=ctrl.Bench.onosInstall,
- name="install-" + ctrl.name,
+ name="onos-install-" + ctrl.name,
kwargs={ "node" : ctrl.ipAddress,
"options" : options } )
threads.append( t )
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index 66eced1..8879220 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -183,6 +183,23 @@
# main.scale[ 0 ] determines the current number of ONOS controller
main.Cluster.setRunningNode( int( main.scale.pop( 0 ) ) )
+ def killingAllAtomix( self, cluster, killRemoveMax, stopAtomix ):
+ """
+ Description:
+ killing atomix. It will either kill the current runningnodes or
+ max number of the nodes.
+ Required:
+ * cluster - the cluster driver that will be used.
+ * killRemoveMax - The boolean that will decide either to kill
+ only running nodes ( False ) or max number of nodes ( True ).
+ * stopAtomix - If wish to stop atomix before killing it. True for
+ enable stop, False for disable stop.
+ Returns:
+ Returns main.TRUE if successfully killing it.
+ """
+ main.log.info( "Safety check, killing all Atomix processes" )
+ return cluster.killAtomix( killRemoveMax, stopAtomix )
+
def killingAllOnos( self, cluster, killRemoveMax, stopOnos ):
"""
Description:
@@ -193,13 +210,12 @@
* killRemoveMax - The boolean that will decide either to kill
only running nodes ( False ) or max number of nodes ( True ).
* stopOnos - If wish to stop onos before killing it. True for
- enable stop , False for disable stop.
+ enable stop, False for disable stop.
Returns:
Returns main.TRUE if successfully killing it.
"""
main.log.info( "Safety check, killing all ONOS processes" )
-
- return cluster.kill( killRemoveMax, stopOnos )
+ return cluster.killOnos( killRemoveMax, stopOnos )
def createApplyCell( self, cluster, newCell, cellName, cellApps,
mininetIp, useSSH, ips, installMax=False ):
@@ -229,6 +245,25 @@
onfail="Failed to apply cell to environment" )
return stepResult
+ def uninstallAtomix( self, cluster, uninstallMax ):
+ """
+ Description:
+ uninstalling atomix and verify the result.
+ Required:
+ * cluster - a cluster driver that will be used.
+ * uninstallMax - True for uninstalling max number of nodes
+ False for uninstalling the current running nodes.
+ Returns:
+ Returns main.TRUE if it successfully uninstalled.
+ """
+ main.step( "Uninstalling Atomix" )
+ atomixUninstallResult = cluster.uninstallAtomix( uninstallMax )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=atomixUninstallResult,
+ onpass="Successfully uninstalled Atomix",
+ onfail="Failed to uninstall Atomix" )
+ return atomixUninstallResult
+
def uninstallOnos( self, cluster, uninstallMax ):
"""
Description:
@@ -241,7 +276,7 @@
Returns main.TRUE if it successfully uninstalled.
"""
main.step( "Uninstalling ONOS package" )
- onosUninstallResult = cluster.uninstall( uninstallMax )
+ onosUninstallResult = cluster.uninstallOnos( uninstallMax )
utilities.assert_equals( expect=main.TRUE,
actual=onosUninstallResult,
onpass="Successfully uninstalled ONOS package",
@@ -265,6 +300,29 @@
onfail="Failed to create ONOS package" )
return packageResult
+ def installAtomix( self, cluster, installMax, parallel=True ):
+ """
+ Description:
+ Installing atomix and verify the result
+ Required:
+ * cluster - the cluster driver that will be used.
+ * installMax - True for installing max number of nodes
+ False for installing current running nodes only.
+ Returns:
+ Returns main.TRUE if it successfully installed
+ """
+ main.step( "Installing Atomix" )
+ atomixInstallResult = main.TRUE
+
+ cluster.installAtomix( installMax, parallel )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=atomixInstallResult,
+ onpass="Successfully installed Atomix",
+ onfail="Failed to install Atomix" )
+ if not atomixInstallResult:
+ main.cleanAndExit()
+ return atomixInstallResult
+
def installOnos( self, cluster, installMax, parallel=True ):
"""
Description:
@@ -279,7 +337,7 @@
main.step( "Installing ONOS package" )
onosInstallResult = main.TRUE
- cluster.install( installMax, parallel )
+ cluster.installOnos( installMax, parallel )
utilities.assert_equals( expect=main.TRUE,
actual=onosInstallResult,
onpass="Successfully installed ONOS package",
@@ -359,7 +417,8 @@
def ONOSSetUp( self, cluster, hasMultiNodeRounds=False, startOnos=True, newCell=True,
cellName="temp", cellApps="drivers", mininetIp="", removeLog=False, extraApply=None, applyArgs=None,
extraClean=None, cleanArgs=None, skipPack=False, installMax=False, useSSH=True,
- killRemoveMax=True, stopOnos=False, installParallel=True, cellApply=True, includeCaseDesc=True ):
+ killRemoveMax=True, stopAtomix=False, stopOnos=False, installParallel=True, cellApply=True,
+ includeCaseDesc=True ):
"""
Description:
Initial ONOS setting up of the tests. It will also verify the result of each steps.
@@ -394,6 +453,7 @@
* useSSH - True for using ssh when creating a cell
* killRemoveMax - True for removing/killing max number of nodes. False for
removing/killing running nodes only.
+ * stopAtomix - True if wish to stop atomix before killing it.
* stopOnos - True if wish to stop onos before killing it.
Returns:
Returns main.TRUE if it everything successfully proceeded.
@@ -404,7 +464,9 @@
" node(s) ONOS cluster" )
main.caseExplanation = "Set up ONOS with " + str( cluster.numCtrls ) + \
" node(s) ONOS cluster"
- killResult = self.killingAllOnos( cluster, killRemoveMax, stopOnos )
+ atomixKillResult = self.killingAllAtomix( cluster, killRemoveMax, stopAtomix )
+ onosKillResult = self.killingAllOnos( cluster, killRemoveMax, stopOnos )
+ killResult = atomixKillResult and onosKillResult
main.log.info( "NODE COUNT = " + str( cluster.numCtrls ) )
cellResult = main.TRUE
@@ -428,13 +490,17 @@
if removeLog:
main.log.info("Removing raft logs")
main.ONOSbench.onosRemoveRaftLogs()
+ atomixUninstallResult = self.uninstallAtomix( cluster, killRemoveMax )
onosUninstallResult = self.uninstallOnos( cluster, killRemoveMax )
+ uninstallResult = atomixUninstallResult and onosUninstallResult
self.processList( extraApply, applyArgs )
if not skipPack:
packageResult = self.buildOnos(cluster)
+ atomixInstallResult = self.installAtomix( cluster, installMax, installParallel )
onosInstallResult = self.installOnos( cluster, installMax, installParallel )
+ installResult = atomixInstallResult and onosInstallResult
self.processList( extraClean, cleanArgs )
secureSshResult = self.setupSsh( cluster )
@@ -444,5 +510,5 @@
if startOnos:
onosCliResult = self.startOnosClis( cluster )
- return killResult and cellResult and packageResult and onosUninstallResult and \
- onosInstallResult and secureSshResult and onosServiceResult and onosCliResult
+ return killResult and cellResult and packageResult and uninstallResult and \
+ installResult and secureSshResult and onosServiceResult and onosCliResult