Initial implementation of ONOS cluster driver
- Create CLI, REST, and "Bench" components for a cluster
- Return driver object when it is created
- Add __str__ and __repr__ implementations for drivers
- Add first pass at a cluster class
- Prototype with clustered Sample test
- Prototype with HAsanity test
- Add new Exception class for SkipCase
Change-Id: I32ee7cf655ab9a2a5cfccf5f891ca71a6a70c1ee
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
new file mode 100644
index 0000000..01dc767
--- /dev/null
+++ b/TestON/tests/dependencies/Cluster.py
@@ -0,0 +1,131 @@
+"""
+Copyright 2017 Open Networking Foundation (ONF)
+
+Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
+the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
+or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
+
+ TestON is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 2 of the License, or
+ (at your option) any later version.
+
+ TestON is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with TestON. If not, see <http://www.gnu.org/licenses/>.
+"""
+
+
+class Cluster():
+
+ def __str__( self ):
+ return self.name
+ def __repr__( self ):
+ #TODO use repr of cli's?
+ controllers = []
+ for ctrl in self.controllers:
+ controllers.append( str( ctrl ) )
+ return "%s[%s]" % ( self.name, ", ".join( controllers ) )
+
+
+ def __init__( self, ctrlList=[], name="Cluster" ):
+ #assert isInstance( ctrlList, Controller ), "ctrlList should be a list of ONOS Controllers"
+ self.controllers = ctrlList
+ self.name = str( name )
+ self.iterator = iter( self.active() )
+
+ def getIps( self, activeOnly=False):
+ ips = []
+ if activeOnly:
+ nodeList = self.active()
+ else:
+ nodeList = self.controllers
+ for ctrl in nodeList:
+ ips.append( ctrl.ipAddress )
+ return ips
+
+ def active( self ):
+ """
+ Return a list of active controllers in the cluster
+ """
+ return [ ctrl for ctrl in self.controllers
+ if ctrl.active ]
+
+ def next( self ):
+ """
+ An iterator for the cluster's controllers that
+ resets when there are no more elements.
+
+ Returns the next controller in the cluster
+ """
+ try:
+ return self.iterator.next()
+ except StopIteration:
+ self.reset()
+ try:
+ return self.iterator.next()
+ except StopIteration:
+ raise RuntimeError( "There are no active nodes in the cluster" )
+
+ def reset( self ):
+ """
+ Resets the cluster iterator.
+
+ This should be used whenever a node's active state is changed
+ and is also used internally when the iterator has been exhausted.
+ """
+ self.iterator = iter( self.active() )
+
+ def install( self ):
+ """
+ Install ONOS on all controller nodes in the cluster
+ """
+ result = main.TRUE
+ # FIXME: use the correct onosdriver function
+ # TODO: Use threads to install in parallel, maybe have an option for sequential installs
+ for ctrl in self.controllers:
+ result &= ctrl.installOnos( ctrl.ipAddress )
+ return result
+
+ def startCLIs( self ):
+ """
+ Start the ONOS cli on all controller nodes in the cluster
+ """
+ cliResults = main.TRUE
+ threads = []
+ for ctrl in self.controllers:
+ t = main.Thread( target=ctrl.CLI.startOnosCli,
+ name="startCli-" + ctrl.name,
+ args=[ ctrl.ipAddress ] )
+ threads.append( t )
+ t.start()
+ ctrl.active = True
+
+ for t in threads:
+ t.join()
+ cliResults = cliResults and t.result
+ return cliResults
+
+ def command( self, function, args=(), kwargs={} ):
+ """
+ Send a command to all ONOS nodes and return the results as a list
+ """
+ threads = []
+ results = []
+ for ctrl in self.active():
+ f = getattr( ctrl, function )
+ t = main.Thread( target=f,
+ name=function + "-" + ctrl.name,
+ args=args,
+ kwargs=kwargs )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ results.append( t.result )
+ return results
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index 70fe3c2..bf3bfd1 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -38,83 +38,34 @@
else:
main.log.info( "Skipped git checkout and pull as they are disabled in params file" )
- return main.TRUE
-
- def setRest( self, hasRest, i ):
- if hasRest:
- main.RESTs.append( getattr( main, "ONOSrest" + str( i ) ) )
-
- def setNode( self, hasNode, i ):
- if hasNode:
- main.nodes.append( getattr( main, 'ONOS' + str(i)) )
-
- def setCli( self, hasCli, i ):
- if hasCli:
- main.CLIs.append( getattr ( main, "ONOScli" + str( i ) ) )
-
- def getNumNode( self, hasCli, hasNode, hasRest ):
- if hasCli:
- return len( main.CLIs )
- if hasNode:
- return len( main.nodes )
- if hasRest:
- return len( main.RESTs )
-
- def envSetup ( self, hasMultiNodeRounds=False, hasRest=False, hasNode=False,
- hasCli=True, specificIp="", includeGitPull=True, makeMaxNodes=True ):
+ def envSetup( self, cluster, hasMultiNodeRounds=False, hasRest=False, hasNode=False,
+ hasCli=True, specificIp="", includeGitPull=True ):
if includeGitPull :
self.gitPulling()
- if main.ONOSbench.maxNodes:
- main.maxNodes = int( main.ONOSbench.maxNodes )
+
+ ONOSbench = cluster.controllers[0].Bench
+ if ONOSbench.maxNodes:
+ main.maxNodes = int( ONOSbench.maxNodes )
else:
main.maxNodes = 0
main.cellData = {} # For creating cell file
- if hasCli:
- main.CLIs = []
- if hasRest:
- main.RESTs = []
- if hasNode:
- main.nodes = []
- main.ONOSip = [] # List of IPs of active ONOS nodes. CASE 2
+ main.ONOSip = cluster.getIps() # List of IPs of active ONOS nodes. CASE 2
- if specificIp == "":
- if makeMaxNodes:
- main.ONOSip = main.ONOSbench.getOnosIps()
- else:
+ # FIXME: Do we need this?
+ # We should be able to just use Cluster.getIps()
+ if specificIp != "":
main.ONOSip.append( specificIp )
# Assigning ONOS cli handles to a list
- try:
- for i in range( 1, ( main.maxNodes if makeMaxNodes else main.numCtrls ) + 1 ):
- self.setCli( hasCli, i )
- self.setRest( hasRest, i )
- self.setNode( hasNode, i )
- if not makeMaxNodes:
- main.ONOSip.append( main.ONOSbench.getOnosIps()[ i - 1 ] )
- except AttributeError:
- numNode = self.getNumNode( hasCli, hasNode, hasRest )
- main.log.warn( "A " + str( main.maxNodes ) + " node cluster " +
- "was defined in env variables, but only " +
- str( numNode ) +
- " nodes were defined in the .topo file. " +
- "Using " + str( numNode ) +
- " nodes for the test." )
- main.maxNodes = numNode
+ main.maxNodes = len( cluster.controllers )
+ return main.TRUE
- main.log.debug( "Found ONOS ips: {}".format( main.ONOSip ) )
- if ( not hasCli or main.CLIs ) and ( not hasRest or main.RESTs )\
- and ( not hasNode or main.nodes ):
- return main.TRUE
- else:
- main.log.error( "Did not properly created list of ONOS CLI handle" )
- return main.FALSE
-
- def envSetupException ( self, e ):
+ def envSetupException( self, e ):
main.log.exception( e )
main.cleanup()
main.exit()
- def evnSetupConclusion ( self, stepResult ):
+ def evnSetupConclusion( self, stepResult ):
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
onpass="Successfully construct " +
@@ -244,22 +195,9 @@
onfail="ONOS service did not start properly on all nodes" )
return stepResult
- def startOnosClis( self ):
- startCliResult = main.TRUE
+ def startOnosClis( self, cluster ):
main.step( "Starting ONOS CLI sessions" )
- pool = []
- main.threadID = 0
- for i in range( main.numCtrls ):
- t = main.Thread( target=main.CLIs[ i ].startOnosCli,
- threadID=main.threadID,
- name="startOnosCli-" + str( i ),
- args=[ main.ONOSip[ i ] ] )
- pool.append( t )
- t.start()
- main.threadID = main.threadID + 1
- for t in pool:
- t.join()
- startCliResult = startCliResult and t.result
+ startCliResult = cluster.startCLIs()
if not startCliResult:
main.log.info( "ONOS CLI did not start up properly" )
main.cleanup()
@@ -272,7 +210,7 @@
onfail="Failed to start ONOS cli" )
return startCliResult
- def ONOSSetUp( self, Mininet, hasMultiNodeRounds=False, hasCli=True, newCell=True,
+ def ONOSSetUp( self, Mininet, cluster, hasMultiNodeRounds=False, hasCli=True, newCell=True,
cellName="temp", removeLog=False, extraApply=None, arg=None, extraClean=None,
skipPack=False, installMax=False, useSSH=True, killRemoveMax=True,
CtrlsSet=True, stopOnos=False ):
@@ -315,7 +253,7 @@
onosServiceResult = self.checkOnosService()
if hasCli:
- onosCliResult = self.startOnosClis()
+ onosCliResult = self.startOnosClis( cluster )
return cellResult and packageResult and onosUninstallResult and \
- onosInstallResult and secureSshResult and onosServiceResult and onosCliResult
\ No newline at end of file
+ onosInstallResult and secureSshResult and onosServiceResult and onosCliResult
diff --git a/TestON/tests/dependencies/topology.py b/TestON/tests/dependencies/topology.py
index f9ce3ff..7819fec 100644
--- a/TestON/tests/dependencies/topology.py
+++ b/TestON/tests/dependencies/topology.py
@@ -15,10 +15,10 @@
"""
devices = []
threads = []
- for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
- t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].devices,
- name="devices-" + str( i ),
- args=[main.CLIs[ i ].devices, [ None ] ] if needRetry else [],
+ for ctrl in main.Cluster.active():
+ t = main.Thread( target=utilities.retry if needRetry else ctrl.devices,
+ name="devices-" + str( ctrl ),
+ args=[ ctrl.devices, [ None ] ] if needRetry else [],
kwargs=kwargs )
threads.append( t )
t.start()
@@ -35,10 +35,10 @@
hosts = []
ipResult = main.TRUE
threads = []
- for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
- t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].hosts,
- name="hosts-" + str( i ),
- args=[main.CLIs[ i ].hosts, [ None ] ] if needRetry else [],
+ for ctrl in main.Cluster.active():
+ t = main.Thread( target=utilities.retry if needRetry else ctrl.hosts,
+ name="hosts-" + str( ctrl ),
+ args=[ ctrl.hosts, [ None ] ] if needRetry else [],
kwargs=kwargs )
threads.append( t )
t.start()
@@ -62,10 +62,10 @@
"""
ports = []
threads = []
- for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
- t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].ports,
- name="ports-" + str( i ),
- args=[ main.CLIs[ i ].ports, [ None ] ] if needRetry else [],
+ for ctrl in main.Cluster.active():
+ t = main.Thread( target=utilities.retry if needRetry else ctrl.ports,
+ name="ports-" + str( ctrl ),
+ args=[ ctrl.ports, [ None ] ] if needRetry else [],
kwargs=kwargs )
threads.append( t )
t.start()
@@ -81,11 +81,10 @@
"""
links = []
threads = []
- print numNode
- for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
- t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].links,
- name="links-" + str( i ),
- args=[main.CLIs[ i ].links, [ None ] ] if needRetry else [],
+ for ctrl in main.Cluster.active():
+ t = main.Thread( target=utilities.retry if needRetry else ctrl.links,
+ name="links-" + str( ctrl ),
+ args=[ ctrl.links, [ None ] ] if needRetry else [],
kwargs=kwargs )
threads.append( t )
t.start()
@@ -102,10 +101,10 @@
"""
clusters = []
threads = []
- for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
- t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].clusters,
- name="clusters-" + str( i ),
- args=[main.CLIs[ i ].clusters, [ None ] ] if needRetry else [],
+ for ctrl in main.Cluster.active():
+ t = main.Thread( target=utilities.retry if needRetry else ctrl.clusters,
+ name="clusters-" + str( ctrl ),
+ args=[ ctrl.clusters, [ None ] ] if needRetry else [],
kwargs=kwargs )
threads.append( t )
t.start()
@@ -124,10 +123,10 @@
mnSwitches,
json.loads( devices[ controller ] ),
json.loads( ports[ controller ] ) )
- except(TypeError, ValueError):
+ except ( TypeError, ValueError ):
main.log.error(
- "Could not load json: {0} or {1}".format( str( devices[ controller ] )
- , str( ports[ controller ] ) ) )
+ "Could not load json: {0} or {1}".format( str( devices[ controller ] ),
+ str( ports[ controller ] ) ) )
currentDevicesResult = main.FALSE
else:
currentDevicesResult = main.FALSE
@@ -193,15 +192,15 @@
controllerStr = str( controller + 1 ) # ONOS node number
# Compare Devices
currentDevicesResult = self.compareDevicePort( Mininet, controller,
- mnSwitches,
- devices, ports )
+ mnSwitches,
+ devices, ports )
if not currentDevicesResult:
deviceFails.append( controllerStr )
devicesResults = devicesResults and currentDevicesResult
# Compare Links
currentLinksResult = self.compareBase( links, controller,
- Mininet.compareLinks,
- [ mnSwitches, mnLinks ] )
+ Mininet.compareLinks,
+ [ mnSwitches, mnLinks ] )
if not currentLinksResult:
linkFails.append( controllerStr )
linksResults = linksResults and currentLinksResult
diff --git a/TestON/tests/dependencies/utils.py b/TestON/tests/dependencies/utils.py
index d82ae04..075b9bd 100644
--- a/TestON/tests/dependencies/utils.py
+++ b/TestON/tests/dependencies/utils.py
@@ -21,6 +21,7 @@
"""
Copy the karaf.log files after each testcase cycle
"""
+ # TODO: Also grab the rotated karaf logs
main.log.report( "Copy karaf logs" )
main.case( "Copy karaf logs" )
main.caseExplanation = "Copying the karaf logs to preserve them through" +\
@@ -29,17 +30,14 @@
stepResult = main.TRUE
scpResult = main.TRUE
copyResult = main.TRUE
- for i in range( main.numCtrls ):
- main.node = main.CLIs[ i ]
- ip = main.ONOSip[ i ]
- main.node.ip_address = ip
- scpResult = scpResult and main.ONOSbench.scp( main.node,
+ for ctrl in main.Cluster.controllers:
+ scpResult = scpResult and main.ONOSbench.scp( ctrl.node,
"/opt/onos/log/karaf.log",
"/tmp/karaf.log",
direction="from" )
copyResult = copyResult and main.ONOSbench.cpLogsToDir( "/tmp/karaf.log", main.logdir,
- copyFileName=( "karaf.log.node{0}.cycle{1}".format(
- str( i + 1 ), str( main.cycle ) ) ) )
+ copyFileName=( "karaf.log.{0}.cycle{1}".format(
+ str( ctrl ), str( main.cycle ) ) ) )
if scpResult and copyResult:
stepResult = main.TRUE and stepResult
else:
@@ -47,4 +45,4 @@
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
onpass="Successfully copied remote ONOS logs",
- onfail="Failed to copy remote ONOS logs" )
\ No newline at end of file
+ onfail="Failed to copy remote ONOS logs" )