Update HA tests to Atomix 3.x
Change-Id: Ic6bb2e4bbd90647f5876a2304ee7b8482673c1c4
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
index 29966d6..6d82489 100644
--- a/TestON/tests/dependencies/Cluster.py
+++ b/TestON/tests/dependencies/Cluster.py
@@ -26,10 +26,16 @@
def __repr__( self ):
controllers = []
- runningNodes = []
+ runningNodes = self.getRunningNodes()
+ atomixNodes = []
for ctrl in self.controllers:
- controllers.append( str( ctrl ) )
- return "%s[%s]" % ( self.name, ", ".join( controllers ) )
+ controllers.append( "{%s:%s, %s - %s}" % ( ctrl.name,
+ ctrl.ipAddress,
+ "Configured" if ctrl in runningNodes else "Not Configured",
+ "Active" if ctrl.active else "Inactive" ) )
+ for node in self.atomixNodes:
+ atomixNodes.append( "{%s:%s}" % ( node.name, node.ipAddress ) )
+ return "%s[%s; Atomix Nodes:%s]" % ( self.name, ", ".join( controllers ), ", ".join( atomixNodes ) )
def __init__( self, ctrlList=[], name="Cluster" ):
"""
@@ -45,8 +51,33 @@
self.numCtrls = len( self.runningNodes )
self.maxCtrls = len( self.controllers )
self.name = str( name )
+ self.atomixNodes = ctrlList
self.iterator = iter( self.active() )
+ def fromNode( self, ctrlList ):
+ """
+ Helper function to get a specific list of controllers
+ Required Arguments:
+ * ctrlList - The list of controllers to return. This can be either an index or a keyword
+ Index | Keyword | List returned
+ 0 | "all" | self.controllers
+ 1 | "running" | self.runningNodes
+ 2 | "active | self.active()
+
+ Throws a ValueError exception if ctrlList value is not recognized
+ """
+ # TODO: Add Atomix Nodes?
+ if isinstance( ctrlList, str ):
+ ctrlList = ctrlList.lower()
+ if ctrlList == 0 or ctrlList == "all":
+ return self.controllers
+ elif ctrlList == 1 or ctrlList == "running":
+ return self.runningNodes
+ elif ctrlList == 2 or ctrlList == "active":
+ return self.active()
+ else:
+ raise ValueError( "Unknown argument: {}".format( ctrlList ) )
+
def getIps( self, activeOnly=False, allNode=False ):
"""
Description:
@@ -108,15 +139,58 @@
self.runningNodes.append( self.controllers[ i ] )
self.numCtrls = len( numCtrls ) if isinstance( numCtrls, list ) else numCtrls
+ def setAtomixNodes( self, nodes ):
+ """
+ Description:
+ Sets the list of Atomix nodes for the cluster
+ If nodes is a list, it will add the nodes of the list.
+ If nodes is an int, the function will set the Atomix nodes
+ to be the first n in the list of controllers.
+ Required:
+ * nodes - number of nodes to be set, or a list of nodes to set
+ Returns:
+ """
+ self.atomixNodes = []
+ for i in nodes if isinstance( nodes, list ) else range( nodes ):
+ self.atomixNodes.append( self.controllers[ i ] )
+
+ def getControllers( self, node=None ):
+ """
+ Description:
+ Get the list of all controllers in a cluster or a controller at an index in the list
+ Optional Arguments:
+ * node - position of the node to get from the list of controllers.
+ Returns:
+ Return a list of controllers in the cluster if node is None
+ if not, it will return the controller at the given index.
+ """
+ result = self.controllers
+ return result if node is None else result[ node % len( result ) ]
+
+ def getRunningNodes( self, node=None ):
+ """
+ Description:
+ Get the list of all controllers in a cluster that should be running or
+ a controller at an index in the list
+ Optional Arguments:
+ * node - position of the node to get from the list of controllers.
+ Returns:
+ Return a list of controllers in the cluster if node is None
+ if not, it will return the controller at the given index.
+ """
+ result = self.runningNodes
+ return result if node is None else result[ node % len( result ) ]
+
def active( self, node=None ):
"""
Description:
- get the list/controller of the active controller.
- Required:
- * node - position of the node to get from the active controller.
+ Get the list of all active controllers in a cluster or
+ a controller at an index in the list
+ Optional Arguments:
+ * node - position of the node to get from the list of controllers.
Returns:
- Return a list of active controllers in the cluster if node is None
- if not, it will return the nth controller.
+ Return a list of controllers in the cluster if node is None
+ if not, it will return the controller at the given index.
"""
result = [ ctrl for ctrl in self.runningNodes
if ctrl.active ]
@@ -149,7 +223,8 @@
"""
self.iterator = iter( self.active() )
- def createCell( self, cellName, cellApps, mininetIp, useSSH, ips, installMax=False ):
+ def createCell( self, cellName, cellApps, mininetIp, useSSH, onosIps,
+ atomixIps, installMax=False ):
"""
Description:
create a new cell
@@ -158,7 +233,9 @@
* cellApps - The ONOS apps string.
* mininetIp - Mininet IP address.
* useSSH - True for using ssh when creating a cell
- * ips - ip( s ) of the node( s ).
+ * onosIps - ip( s ) of the ONOS node( s ).
+ * atomixIps - ip( s ) of the Atomix node( s ).
+
Returns:
"""
self.command( "createCellFile",
@@ -166,11 +243,12 @@
cellName,
mininetIp,
cellApps,
- ips,
+ onosIps,
+ atomixIps,
main.ONOScell.karafUser,
useSSH ],
specificDriver=1,
- getFrom=0 if installMax else 1 )
+ getFrom="all" if installMax else "running" )
def uninstallAtomix( self, uninstallMax ):
"""
@@ -186,7 +264,7 @@
uninstallResult = self.command( "atomixUninstall",
kwargs={ "nodeIp": "ipAddress" },
specificDriver=1,
- getFrom=0 if uninstallMax else 1,
+ getFrom="all" if uninstallMax else "running",
funcFromCtrl=True )
for uninstallR in uninstallResult:
result = result and uninstallR
@@ -206,7 +284,7 @@
uninstallResult = self.command( "onosUninstall",
kwargs={ "nodeIp": "ipAddress" },
specificDriver=1,
- getFrom=0 if uninstallMax else 1,
+ getFrom="all" if uninstallMax else "running",
funcFromCtrl=True )
for uninstallR in uninstallResult:
result = result and uninstallR
@@ -225,13 +303,15 @@
setCellResult = self.command( "setCell",
args=[ cellName ],
specificDriver=1,
- getFrom=0 if installMax else 1 )
+ getFrom="all" )
+ benchCellResult = main.ONOSbench.setCell( cellName )
verifyResult = self.command( "verifyCell",
specificDriver=1,
- getFrom=0 if installMax else 1 )
+ getFrom="all" )
result = main.TRUE
for i in range( len( setCellResult ) ):
result = result and setCellResult[ i ] and verifyResult[ i ]
+ result = result and benchCellResult
return result
def checkService( self ):
@@ -243,16 +323,16 @@
Returns:
Returns main.TRUE if it successfully checked
"""
- stopResult = main.TRUE
- startResult = main.TRUE
+ getFrom = "running"
onosIsUp = main.TRUE
onosUp = self.command( "isup",
args=[ "ipAddress" ],
specificDriver=1,
- getFrom=1,
+ getFrom=getFrom,
funcFromCtrl=True )
+ ctrlList = self.fromNode( getFrom )
for i in range( len( onosUp ) ):
- ctrl = self.controllers[ i ]
+ ctrl = ctrlList[ i ]
onosIsUp = onosIsUp and onosUp[ i ]
if onosUp[ i ] == main.TRUE:
main.log.info( ctrl.name + " is up and ready" )
@@ -277,11 +357,10 @@
killResult = self.command( "atomixKill",
args=[ "ipAddress" ],
specificDriver=1,
- getFrom=0 if killMax else 1,
+ getFrom="all" if killMax else "running",
funcFromCtrl=True )
for i in range( len( killResult ) ):
result = result and killResult[ i ]
- self.controllers[ i ].active = False
return result
def killOnos( self, killMax, stopOnos ):
@@ -297,15 +376,17 @@
Returns:
Returns main.TRUE if successfully killing it.
"""
+ getFrom = "all" if killMax else "running"
result = main.TRUE
killResult = self.command( "onosKill",
args=[ "ipAddress" ],
specificDriver=1,
- getFrom=0 if killMax else 1,
+ getFrom=getFrom,
funcFromCtrl=True )
+ ctrlList = self.fromNode( getFrom )
for i in range( len( killResult ) ):
result = result and killResult[ i ]
- self.controllers[ i ].active = False
+ ctrlList[ i ].active = False
return result
def ssh( self ):
@@ -321,30 +402,25 @@
sshResult = self.command( "onosSecureSSH",
kwargs={ "node": "ipAddress" },
specificDriver=1,
- getFrom=1,
+ getFrom="running",
funcFromCtrl=True )
for sshR in sshResult:
result = result and sshR
return result
- def installAtomix( self, installMax=True, installParallel=True ):
+ def installAtomix( self, installParallel=True ):
"""
Description:
- Installing onos.
+ Installing Atomix.
Required:
- * installMax - True for installing max number of nodes
- False for installing current running nodes only.
Returns:
Returns main.TRUE if it successfully installed
"""
result = main.TRUE
threads = []
i = 0
- for ctrl in self.controllers if installMax else self.runningNodes:
- options = ""
- if installMax and i >= self.numCtrls:
- # TODO: is installMax supported here?
- pass
+ for ctrl in self.atomixNodes:
+ options = "-f"
if installParallel:
t = main.Thread( target=ctrl.Bench.atomixInstall,
name="atomix-install-" + ctrl.name,
@@ -404,15 +480,17 @@
Returns:
Returns main.TRUE if it successfully started.
"""
+ getFrom = "running"
result = main.TRUE
cliResults = self.command( "startOnosCli",
args=[ "ipAddress" ],
specificDriver=2,
- getFrom=1,
+ getFrom=getFrom,
funcFromCtrl=True )
+ ctrlList = self.fromNode( getFrom )
for i in range( len( cliResults ) ):
result = result and cliResults[ i ]
- self.controllers[ i ].active = True
+ ctrlList[ i ].active = True
return result
def nodesCheck( self ):
@@ -437,6 +515,8 @@
activeIps.sort()
if ips == activeIps:
currentResult = True
+ else:
+ main.log.error( "{} != {}".format( ips, activeIps ) )
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
@@ -526,7 +606,7 @@
return all( resultOne == result for result in results )
def command( self, function, args=(), kwargs={}, returnBool=False,
- specificDriver=0, contentCheck=False, getFrom=2,
+ specificDriver=0, contentCheck=False, getFrom="active",
funcFromCtrl=False ):
"""
Description:
@@ -546,9 +626,9 @@
* contentCheck - If this is True, it will check if the result has some
contents.
* getFrom - from which nodes
- 2 - active nodes
- 1 - current running nodes
- 0 - all nodes
+ 2 or "active" - active nodes
+ 1 or "running" - current running nodes
+ 0 or "all" - all nodes
* funcFromCtrl - specific function of the args/kwargs
from each controller from the list of the controllers
Returns:
@@ -558,9 +638,8 @@
"""
threads = []
drivers = [ None, "Bench", "CLI", "REST" ]
- fromNode = [ self.controllers, self.runningNodes, self.active() ]
results = []
- for ctrl in fromNode[ getFrom ]:
+ for ctrl in self.fromNode( getFrom ):
try:
funcArgs = []
funcKwargs = {}
@@ -587,9 +666,9 @@
t.join()
results.append( t.result )
if returnBool:
- return self.allTrueResultCheck( results, fromNode[ getFrom ] )
+ return self.allTrueResultCheck( results, self.fromNode( getFrom ) )
elif contentCheck:
- return self.notEmptyResultCheck( results, fromNode[ getFrom ] )
+ return self.notEmptyResultCheck( results, self.fromNode( getFrom ) )
return results
def checkPartitionSize( self, segmentSize='64', units='M', multiplier='3' ):
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index 03328e7..38ba838 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -169,7 +169,7 @@
def setNumCtrls( self, hasMultiNodeRounds ):
"""
Description:
- Set new number of controls if it uses different number of nodes.
+ Set new number of controllers if it uses different number of nodes.
different number of nodes should be pre-defined in main.scale.
Required:
* hasMultiNodeRouds - if the test is testing different number of nodes.
@@ -180,7 +180,7 @@
except Exception:
main.cycle = 0
main.cycle += 1
- # main.scale[ 0 ] determines the current number of ONOS controller
+ # main.scale[ 0 ] determines the current number of ONOS controllers
main.Cluster.setRunningNode( int( main.scale.pop( 0 ) ) )
def killingAllAtomix( self, cluster, killRemoveMax, stopAtomix ):
@@ -218,24 +218,36 @@
return cluster.killOnos( killRemoveMax, stopOnos )
def createApplyCell( self, cluster, newCell, cellName, cellApps,
- mininetIp, useSSH, ips, installMax=False ):
+ mininetIp, useSSH, onosIps, installMax=False,
+ atomixClusterSize=None ):
"""
Description:
create new cell ( optional ) and apply it. It will also verify the
cell.
- Required:
+ Required Arguments:
* cluster - the cluster driver that will be used.
* newCell - True for making a new cell and False for not making it.
* cellName - The name of the cell.
* cellApps - The onos apps string.
* mininetIp - Mininet IP address.
* useSSH - True for using ssh when creating a cell
- * ips - ip( s ) of the node( s ).
+ * onosIps - ip( s ) of the ONOS node( s ).
+ Optional Arguments:
+ * installMax
+ * atomixClusterSize - The size of the atomix cluster. Defaults to same
+ as ONOS Cluster size
Returns:
Returns main.TRUE if it successfully executed.
"""
+ if atomixClusterSize is None:
+ atomixClusterSize = len( cluster.runningNodes )
+ atomixClusterSize = int( atomixClusterSize )
+ cluster.setAtomixNodes( atomixClusterSize )
+ atomixIps = [ node.ipAddress for node in cluster.atomixNodes ]
+ main.log.info( "Atomix Cluster Size = {} ".format( atomixClusterSize ) )
if newCell:
- cluster.createCell( cellName, cellApps, mininetIp, useSSH, ips )
+ cluster.createCell( cellName, cellApps, mininetIp, useSSH, onosIps,
+ atomixIps, installMax )
main.step( "Apply cell to environment" )
stepResult = cluster.applyCell( cellName )
utilities.assert_equals( expect=main.TRUE,
@@ -252,7 +264,7 @@
Required:
* cluster - a cluster driver that will be used.
* uninstallMax - True for uninstalling max number of nodes
- False for uninstalling the current running nodes.
+ False for uninstalling the current running nodes.
Returns:
Returns main.TRUE if it successfully uninstalled.
"""
@@ -300,13 +312,12 @@
onfail="Failed to create ONOS package" )
return packageResult
- def installAtomix( self, cluster, installMax, parallel=True ):
+ def installAtomix( self, cluster, parallel=True ):
"""
Description:
Installing atomix and verify the result
Required:
* cluster - the cluster driver that will be used.
- * installMax - True for installing max number of nodes
False for installing current running nodes only.
Returns:
Returns main.TRUE if it successfully installed
@@ -314,7 +325,7 @@
main.step( "Installing Atomix" )
atomixInstallResult = main.TRUE
- cluster.installAtomix( installMax, parallel )
+ cluster.installAtomix( parallel )
utilities.assert_equals( expect=main.TRUE,
actual=atomixInstallResult,
onpass="Successfully installed Atomix",
@@ -360,8 +371,8 @@
secureSshResult = cluster.ssh()
utilities.assert_equals( expect=main.TRUE,
actual=secureSshResult,
- onpass="Test step PASS",
- onfail="Test step FAIL" )
+ onpass="Secured ONOS ssh",
+ onfail="Failed to secure ssh" )
return secureSshResult
def checkOnosService( self, cluster ):
@@ -447,7 +458,7 @@
False,
args = [ apps ],
sleep=5,
- attempts=9 )
+ attempts=90 )
utilities.assert_equals( expect=True,
actual=stepResult,
@@ -468,9 +479,10 @@
def ONOSSetUp( self, cluster, hasMultiNodeRounds=False, startOnos=True, newCell=True,
cellName="temp", cellApps="drivers", appPrefix="org.onosproject.",
- mininetIp="", removeLog=False, extraApply=None, applyArgs=None,
- extraClean=None, cleanArgs=None, skipPack=False, installMax=False, useSSH=True,
- killRemoveMax=True, stopAtomix=False, stopOnos=False, installParallel=True, cellApply=True,
+ mininetIp="", extraApply=None, applyArgs=None,
+ extraClean=None, cleanArgs=None, skipPack=False, installMax=False,
+ atomixClusterSize=None, useSSH=True, killRemoveMax=True, stopAtomix=False,
+ stopOnos=False, installParallel=True, cellApply=True,
includeCaseDesc=True ):
"""
Description:
@@ -487,26 +499,28 @@
setting up ssh to the onos
checking the onos service
starting onos
- Required:
+ Required Arguments:
* cluster - the cluster driver that will be used.
- * hasMultiNodeRouds - True if the test is testing different set of nodes
+ Optional Arguments:
+ * hasMultiNodeRounds - True if the test is testing different set of nodes
* startOnos - True if wish to start onos.
* newCell - True for making a new cell and False for not making it.
* cellName - Name of the cell that will be used.
* cellApps - The cell apps string. Will be overwritten by main.apps if it exists
* appPrefix - Prefix of app names. Will use "org.onosproject." by default
* mininetIp - Mininet IP address.
- * removeLog - True if wish to remove raft logs
* extraApply - Function( s ) that will be called before building ONOS. Default to None.
* applyArgs - argument of the functon( s ) of the extraApply. Should be in list.
* extraClean - Function( s ) that will be called after building ONOS. Defaults to None.
* cleanArgs - argument of the functon( s ) of the extraClean. Should be in list.
* skipPack - True if wish to skip some packing.
* installMax - True if wish to install onos max number of nodes
- False if wish to install onos of running nodes only
+ False if wish to install onos of running nodes only
+ * atomixClusterSize - The number of Atomix nodes in the cluster.
+ Defaults to None which will be the number of OC# nodes in the cell
* useSSH - True for using ssh when creating a cell
* killRemoveMax - True for removing/killing max number of nodes. False for
- removing/killing running nodes only.
+ removing/killing running nodes only.
* stopAtomix - True if wish to stop atomix before killing it.
* stopOnos - True if wish to stop onos before killing it.
Returns:
@@ -519,7 +533,7 @@
main.caseExplanation = "Set up ONOS with " + str( cluster.numCtrls ) + \
" node(s) ONOS cluster"
- main.log.info( "NODE COUNT = " + str( cluster.numCtrls ) )
+ main.log.info( "ONOS cluster size = " + str( cluster.numCtrls ) )
cellResult = main.TRUE
if cellApply:
try:
@@ -539,15 +553,13 @@
cellResult = self.createApplyCell( cluster, newCell,
cellName, apps,
mininetIp, useSSH,
- tempOnosIp, installMax )
+ tempOnosIp, installMax,
+ atomixClusterSize )
atomixKillResult = self.killingAllAtomix( cluster, killRemoveMax, stopAtomix )
onosKillResult = self.killingAllOnos( cluster, killRemoveMax, stopOnos )
killResult = atomixKillResult and onosKillResult
- if removeLog:
- main.log.info("Removing raft logs")
- main.ONOSbench.onosRemoveRaftLogs()
atomixUninstallResult = self.uninstallAtomix( cluster, killRemoveMax )
onosUninstallResult = self.uninstallOnos( cluster, killRemoveMax )
uninstallResult = atomixUninstallResult and onosUninstallResult
@@ -557,7 +569,7 @@
if not skipPack:
packageResult = self.buildOnos(cluster)
- atomixInstallResult = self.installAtomix( cluster, installMax, installParallel )
+ atomixInstallResult = self.installAtomix( cluster, installParallel )
onosInstallResult = self.installOnos( cluster, installMax, installParallel )
installResult = atomixInstallResult and onosInstallResult