Add functions to check the size of raft logs
- Function to check the size of a file/folder using du
- Check the size of the raft logs in HA cleanup function
- Check is based on max segment size of the raft instance(s)
Change-Id: I5259bb8520bbe962c8c2970b77164e6ff2e1000b
diff --git a/TestON/tests/HA/dependencies/HA.py b/TestON/tests/HA/dependencies/HA.py
index f1ddaef..48b64c6 100644
--- a/TestON/tests/HA/dependencies/HA.py
+++ b/TestON/tests/HA/dependencies/HA.py
@@ -136,7 +136,8 @@
main.log.warn( javaArgs )
main.log.warn( repr( javaArgs ) )
handle = main.ONOSbench.handle
- sed = r"sed -i 's/bash/bash\nexport JAVA_OPTS=${{JAVA_OPTS:-{}}}\n/' {}".format( javaArgs, main.onosServicepath )
+ sed = r"sed -i 's/bash/bash\nexport JAVA_OPTS=${{JAVA_OPTS:-{}}}\n/' {}".format( javaArgs,
+ main.onosServicepath )
main.log.warn( sed )
main.log.warn( repr( sed ) )
handle.sendline( sed )
@@ -326,8 +327,6 @@
"""
rest of initialSetup
"""
-
-
if main.params[ 'tcpdump' ].lower() == "true":
main.step( "Start Packet Capture MN" )
main.Mininet2.startTcpdump(
@@ -450,7 +449,8 @@
main.log.exception( "Error parsing leaders" )
main.log.error( repr( leaders ) )
if missing:
- #NOTE Can we refactor this into the Cluster class? Maybe an option to print the output of a command from each node?
+ # NOTE Can we refactor this into the Cluster class?
+ # Maybe an option to print the output of a command from each node?
for ctrl in main.Cluster.active():
response = ctrl.CLI.leaders( jsonFormat=False )
main.log.debug( str( ctrl.name ) + " leaders output: \n" +
@@ -569,7 +569,7 @@
if re.search( "tcp:" + ctrl.ipAddress, response ):
mastershipCheck = mastershipCheck and main.TRUE
else:
- main.log.error( "Error, node " + repr( ctrl )+ " is " +
+ main.log.error( "Error, node " + repr( ctrl ) + " is " +
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
@@ -915,7 +915,7 @@
indent=4,
separators=( ',', ': ' ) )
except ( ValueError, TypeError ):
- output = repr( tmpIntents )
+ output = repr( tmpIntents )
main.log.debug( "ONOS1 intents: " + output )
utilities.assert_equals(
expect=main.TRUE,
@@ -1099,14 +1099,12 @@
mastershipCheck = main.TRUE
mastershipState = ONOSMastership[ 0 ]
-
global intentState
intentState = []
ONOSIntents, intentsResults = self.checkingIntents()
intentCheck = main.FALSE
consistentIntents = True
-
main.step( "Check for consistency in Intents from each controller" )
if all( [ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
main.log.info( "Intents are consistent across all ONOS " +
@@ -1178,7 +1176,7 @@
main.step( "Get the flows from each controller" )
global flowState
flowState = []
- ONOSFlows = main.Cluster.command( "flows", specificDriver=2 ) # TODO: Possible arg: sleep = 30
+ ONOSFlows = main.Cluster.command( "flows", specificDriver=2 ) # TODO: Possible arg: sleep = 30
ONOSFlowsJson = []
flowCheck = main.FALSE
consistentFlows = True
@@ -2025,8 +2023,8 @@
main.step( "Distributed Set clear()" )
main.onosSet.clear()
clearResponses = main.Cluster.command( "setTestRemove",
- args=[ main.onosSetName, " " ], # Values doesn't matter
- kwargs={ "clear": True } )
+ args=[ main.onosSetName, " " ], # Values doesn't matter
+ kwargs={ "clear": True } )
# main.TRUE = successfully changed the set
# main.FALSE = action resulted in no change in set
# main.ERROR - Some error in executing the function
@@ -2536,7 +2534,17 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
+
main.case( "Test Cleanup" )
+
+ main.step( "Checking raft log size" )
+ # TODO: this is a flaky check, but the intent is to make sure the raft logs
+ # get compacted periodically
+ logCheck = main.Cluster.checkPartitionSize()
+ utilities.assert_equals( expect=True, actual=logCheck,
+ onpass="Raft log size is not too big",
+ onfail="Raft logs grew too big" )
+
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2780,11 +2788,11 @@
utilities.assert_equals( expect=main.TRUE, actual=runResults,
onpass="ONOS nodes reran for election topic",
onfail="Errror rerunning for election" )
+
def tempCell( self, cellName, ipList ):
main.step( "Create cell file" )
cellAppString = main.params[ 'ENV' ][ 'appString' ]
-
main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
main.Mininet1.ip_address,
cellAppString, ipList , main.ONOScli1.karafUser )
@@ -2792,7 +2800,6 @@
cellResult = main.ONOSbench.setCell( cellName )
verifyResult = main.ONOSbench.verifyCell()
-
def checkStateAfterEvent( self, main, afterWhich, compareSwitch=False, isRestart=False ):
"""
afterWhich :
@@ -3108,9 +3115,11 @@
mnHosts = main.Mininet1.getHosts()
for controller in range( len( main.Cluster.active() ) ):
controllerStr = str( main.Cluster.active( controller ) )
- currentDevicesResult = main.topoRelated.compareDevicePort( main.Mininet1, controller,
- mnSwitches,
- devices, ports )
+ currentDevicesResult = main.topoRelated.compareDevicePort( main.Mininet1,
+ controller,
+ mnSwitches,
+ devices,
+ ports )
utilities.assert_equals( expect=main.TRUE,
actual=currentDevicesResult,
onpass=controllerStr +
@@ -3118,10 +3127,9 @@
onfail=controllerStr +
" Switches view is incorrect" )
-
currentLinksResult = main.topoRelated.compareBase( links, controller,
- main.Mininet1.compareLinks,
- [mnSwitches, mnLinks] )
+ main.Mininet1.compareLinks,
+ [ mnSwitches, mnLinks ] )
utilities.assert_equals( expect=main.TRUE,
actual=currentLinksResult,
onpass=controllerStr +
@@ -3428,7 +3436,7 @@
"working properly"
main.case( description )
- main.step( "Bring link between " + fromS + " and " + toS +" back up" )
+ main.step( "Bring link between " + fromS + " and " + toS + " back up" )
LinkUp = main.Mininet1.link( END1=fromS, END2=toS, OPTION="up" )
main.log.info( "Waiting " + str( linkSleep ) +
" seconds for link up to be discovered" )
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
index 5c6e679..16dfd79 100644
--- a/TestON/tests/dependencies/Cluster.py
+++ b/TestON/tests/dependencies/Cluster.py
@@ -24,15 +24,14 @@
def __str__( self ):
return self.name
+
def __repr__( self ):
- #TODO use repr of cli's?
controllers = []
runningNodes = []
for ctrl in self.controllers:
controllers.append( str( ctrl ) )
return "%s[%s]" % ( self.name, ", ".join( controllers ) )
-
def __init__( self, ctrlList=[], name="Cluster" ):
'''
controllers : All the nodes
@@ -186,7 +185,7 @@
"""
result = main.TRUE
uninstallResult = self.command( "onosUninstall",
- kwargs={ "nodeIp":"ipAddress" },
+ kwargs={ "nodeIp": "ipAddress" },
specificDriver=1,
getFrom=0 if uninstallMax else 1,
funcFromCtrl=True )
@@ -206,12 +205,12 @@
"""
setCellResult = self.command( "setCell",
- args=[ cellName ],
- specificDriver=1,
- getFrom=0 if installMax else 1 )
+ args=[ cellName ],
+ specificDriver=1,
+ getFrom=0 if installMax else 1 )
verifyResult = self.command( "verifyCell",
- specificDriver=1,
- getFrom=0 if installMax else 1 )
+ specificDriver=1,
+ getFrom=0 if installMax else 1 )
result = main.TRUE
for i in range( len( setCellResult ) ):
result = result and setCellResult[ i ] and verifyResult[ i ]
@@ -267,7 +266,7 @@
specificDriver=1,
getFrom=0 if killMax else 1,
funcFromCtrl=True )
- for i in range( len ( killResult ) ):
+ for i in range( len( killResult ) ):
result = result and killResult[ i ]
self.controllers[ i ].active = False
return result
@@ -283,7 +282,7 @@
"""
result = main.TRUE
sshResult = self.command( "onosSecureSSH",
- kwargs={ "node":"ipAddress" },
+ kwargs={ "node": "ipAddress" },
specificDriver=1,
getFrom=1,
funcFromCtrl=True )
@@ -309,10 +308,10 @@
if installMax and i >= self.numCtrls:
options = "-nf"
if installParallel:
- t= main.Thread( target=ctrl.Bench.onosInstall,
- name="install-" + ctrl.name,
- kwargs={ "node" : ctrl.ipAddress,
- "options" : options } )
+ t = main.Thread( target=ctrl.Bench.onosInstall,
+ name="install-" + ctrl.name,
+ kwargs={ "node" : ctrl.ipAddress,
+ "options" : options } )
threads.append( t )
t.start()
else:
@@ -338,7 +337,7 @@
specificDriver=2,
getFrom=1,
funcFromCtrl=True )
- for i in range ( len( cliResults ) ):
+ for i in range( len( cliResults ) ):
result = result and cliResults[ i ]
self.controllers[ i ].active = True
return result
@@ -354,8 +353,8 @@
Returns:
"""
f = getattr( main.log, logLevel )
- for i in range ( len ( results ) ):
- f( activeList[ i ].name + "'s result : " + str ( results[ i ] ) )
+ for i in range( len( results ) ):
+ f( activeList[ i ].name + "'s result : " + str( results[ i ] ) )
def allTrueResultCheck( self, results, activeList ):
"""
@@ -447,7 +446,7 @@
funcArgs.append( getattr( ctrl, args[ i ] ) )
if kwargs:
for k in kwargs:
- funcKwargs.update( { k:getattr( ctrl, kwargs[ k ] ) } )
+ funcKwargs.update( { k: getattr( ctrl, kwargs[ k ] ) } )
except AttributeError:
main.log.error( "Function " + function + " not found. Exiting the Test." )
main.cleanAndExit()
@@ -465,4 +464,16 @@
return self.allTrueResultCheck( results, fromNode[ getFrom ] )
elif contentCheck:
return self.notEmptyResultCheck( results, fromNode[ getFrom ] )
- return results
\ No newline at end of file
+ return results
+
+ def checkPartitionSize( self, segmentSize='64', units='M', multiplier='3' ):
+ # max segment size in bytes: 1024 * 1024 * 64
+ # multiplier is somewhat arbitrary, but the idea is the logs would have
+ # been compacted before this many segments are written
+
+ maxSize = float( segmentSize ) * float( multiplier )
+ ret = True
+ for n in self.runningNodes:
+ ret = ret and n.server.folderSize( "/opt/onos/apache-karaf-*/data/partitions/*/*.log",
+ size=maxSize, unit=units, ignoreRoot=False )
+ return ret