blob: 566e02b663a064d01ad29b5197acdd86249917a7 [file] [log] [blame]
"""
Copyright 2015 Open Networking Foundation ( ONF )
Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
TestON is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
( at your option ) any later version.
TestON is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TestON. If not, see <http://www.gnu.org/licenses/>.
"""
"""
SCPFintentInstallWithdrawLat:
- Test the latency of intent installed and withdrawn
- Use Push-test-intents command to push intents
- Use Null provider with 7 devices and linear topology
- Always push intents between 1/6 and 7/5
- The batch size is defined in parm file. ( default 1,100,1000 )
yunpeng@onlab.us
"""
class SCPFintentInstallWithdrawLat:
def __init__( self ):
self.default = ''
def CASE0( self, main ):
import imp
import os
"""
- GIT
- BUILDING ONOS
Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
This step is usually skipped. Because in a Jenkins driven automated
test env. We want Jenkins jobs to pull&build for flexibility to handle
different versions of ONOS.
- Construct tests variables
"""
try:
from tests.dependencies.ONOSSetup import ONOSSetup
main.testSetUp = ONOSSetup()
except ImportError:
main.log.error( "ONOSSetup not found. exiting the test" )
main.cleanAndExit()
main.testSetUp.envSetupDescription()
stepResult = main.FALSE
try:
main.apps = main.params[ 'ENV' ][ 'cellApps' ]
main.BENCHUser = main.params[ 'BENCH' ][ 'user' ]
main.BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
main.MN1Ip = main.params[ 'MN' ][ 'ip1' ]
main.maxNodes = int( main.params[ 'max' ] )
main.cellName = main.params[ 'ENV' ][ 'cellName' ]
main.scale = ( main.params[ 'SCALE' ] ).split( "," )
main.timeout = int( main.params[ 'SLEEP' ][ 'timeout' ] )
main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] )
main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] )
main.intentManagerCfg = main.params[ 'CFG' ][ 'intentManager' ]
main.intentConfigRegiCfg = main.params[ 'CFG' ][ 'intentConfigRegi' ]
main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ]
main.verifyAttempts = int( main.params[ 'ATTEMPTS' ][ 'verify' ] )
main.cfgRetry = int( main.params[ 'ATTEMPTS' ][ 'cfg' ] )
main.maxInvalidRun = int( main.params[ 'ATTEMPTS' ][ 'maxInvalidRun' ] )
main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
main.intentsList = ( main.params[ 'TEST' ][ 'intents' ] ).split( "," )
main.deviceCount = int( main.params[ 'TEST' ][ 'deviceCount' ] )
main.ingress = main.params[ 'TEST' ][ 'ingress' ]
main.egress = main.params[ 'TEST' ][ 'egress' ]
main.debug = main.params[ 'TEST' ][ 'debug' ]
main.flowObj = main.params[ 'TEST' ][ 'flowObj' ]
if main.flowObj == "True":
main.flowObj = True
main.dbFileName = main.params[ 'DATABASE' ][ 'dbFlowObj' ]
else:
main.flowObj = False
main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ]
for i in range( 0, len( main.intentsList ) ):
main.intentsList[ i ] = int( main.intentsList[ i ] )
stepResult = main.testSetUp.envSetup()
# Create DataBase file
main.log.info( "Create Database file " + main.dbFileName )
resultsDB = open( main.dbFileName, "w+" )
resultsDB.close()
file1 = main.params[ "DEPENDENCY" ][ "FILE1" ]
main.dependencyPath = os.path.dirname( os.getcwd() ) + main.params[ "DEPENDENCY" ][ "PATH" ]
main.intentFuncs = imp.load_source( file1, main.dependencyPath + file1 + ".py" )
except Exception as e:
main.testSetUp.envSetupException( e )
main.testSetUp.envSetupConclusion( stepResult )
main.commit = main.commit.split( " " )[ 1 ]
def CASE1( self, main ):
# Clean up test environment and set up
import time
main.maxNumBatch = 0
main.testSetUp.ONOSSetUp( main.Cluster, True,
cellName=main.cellName, killRemoveMax=False )
configRetry = 0
main.cfgCheck = False
while configRetry < main.cfgRetry:
# configure apps
stepResult = main.TRUE
stepResult = stepResult and \
main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
"deviceCount", value=main.deviceCount )
stepResult = stepResult and \
main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
"topoShape", value="linear" )
stepResult = stepResult and \
main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
"enabled", value="true" )
stepResult = stepResult and \
main.Cluster.active( 0 ).CLI.setCfg( main.intentManagerCfg,
"skipReleaseResourcesOnWithdrawal",
value="true" )
if main.flowObj:
stepResult = stepResult and \
main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
"useFlowObjectives", value="true" )
time.sleep( main.startUpSleep )
# balanceMasters
stepResult = stepResult and \
main.Cluster.active( 0 ).CLI.balanceMasters()
if stepResult:
main.cfgCheck = True
break
configRetry += 1
time.sleep( main.verifySleep )
time.sleep( main.startUpSleep )
if not main.cfgCheck:
main.log.error( "Setting configuration to the ONOS failed. Skip the rest of the steps" )
def CASE2( self, main ):
import time
import numpy
import json
testResult = main.TRUE
main.case( "Installing/Withdrawing intents start" )
main.step( "Checking starts" )
if main.cfgCheck:
print( main.intentsList )
for batchSize in main.intentsList:
main.log.report( "Intent Batch size: {}".format( batchSize ) )
main.batchSize = batchSize
main.installLatList = []
main.withdrawLatList = []
main.validrun = 0
main.invalidrun = 0
# we use two variables to control the iteration
while main.validrun <= main.warmUp + main.sampleSize and main.invalidrun <= main.maxInvalidRun:
if main.validrun >= main.warmUp:
main.log.info( "================================================" )
main.log.info( "Starting test iteration " + str( main.validrun - main.warmUp ) )
main.log.info( "Total test iteration: " + str( main.invalidrun + main.validrun ) )
main.log.info( "================================================" )
else:
main.log.info( "====================Warm Up=====================" )
# push intents
installResult = main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
main.egress,
batchSize,
offset=1,
options="-i",
timeout=main.timeout,
getResponse=True )
time.sleep( 2 )
main.intentFuncs.sanityCheck( main,
( main.deviceCount - 1 ) * 2,
batchSize * main.deviceCount,
main.batchSize )
if not main.verify:
main.log.warn( "Sanity check failed, skipping this iteration..." )
continue
if isinstance( installResult, str ):
if "Failure" in installResult:
main.log.error( "Install Intents failure, ignore this iteration." )
if main.validrun < main.warmUp:
main.validrun += 1
continue
else:
main.invalidrun += 1
continue
try:
latency = int( installResult.split()[ -2 ] )
main.log.info( installResult )
except:
main.log.error( "Failed to get latency, ignore this iteration." )
main.log.error( "Response from ONOS:" )
print( installResult )
if main.validrun < main.warmUp:
main.validrun += 1
continue
else:
main.invalidrun += 1
continue
if main.validrun >= main.warmUp:
main.installLatList.append( latency )
else:
main.invalidrun += 1
continue
# Withdraw Intents
withdrawResult = main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
main.egress,
batchSize,
offset=1,
options="-w",
timeout=main.timeout,
getResponse=True )
time.sleep( 5 )
main.Cluster.active( 0 ).CLI.purgeWithdrawnIntents()
main.intentFuncs.sanityCheck( main, ( main.deviceCount - 1 ) * 2, 0, 0 )
if not main.verify:
main.log.warn( "Sanity check failed, skipping this iteration..." )
continue
if isinstance( withdrawResult, str ):
if "Failure" in withdrawResult:
main.log.error( "withdraw Intents failure, ignore this iteration." )
if main.validrun < main.warmUp:
main.validrun += 1
continue
else:
main.invalidrun += 1
continue
try:
latency = int( withdrawResult.split()[ -2 ] )
main.log.info( withdrawResult )
except:
main.log.error( "Failed to get latency, ignore this iteration." )
main.log.error( "Response from ONOS:" )
print( withdrawResult )
if main.validrun < main.warmUp:
main.validrun += 1
continue
else:
main.invalidrun += 1
continue
if main.validrun >= main.warmUp:
main.withdrawLatList.append( latency )
else:
main.invalidrun += 1
continue
main.validrun += 1
result = ( main.TRUE if main.invalidrun <= main.maxInvalidRun else main.FALSE )
installave = numpy.average( main.installLatList ) if main.installLatList and result else 0
installstd = numpy.std( main.installLatList ) if main.installLatList and result else 0
withdrawave = numpy.average( main.withdrawLatList ) if main.withdrawLatList and result else 0
withdrawstd = numpy.std( main.withdrawLatList ) if main.withdrawLatList and result else 0
testResult = testResult and result
# log report
main.log.report( "----------------------------------------------------" )
main.log.report( "Scale: " + str( main.Cluster.numCtrls ) )
main.log.report( "Intent batch: " + str( batchSize ) )
main.log.report( "Install average: {} std: {}".format( installave, installstd ) )
main.log.report( "Withdraw average: {} std: {}".format( withdrawave, withdrawstd ) )
# Check test results
index = len( main.intentsList ) * ( main.cycle - 1 ) + main.intentsList.index( batchSize )
# Check installation results
if main.flowObj:
threshold = float( main.params[ 'ALARM' ][ 'maxInstallLatFlowObj' ].split( ',' )[ index ])
else:
threshold = float( main.params[ 'ALARM' ][ 'maxInstallLat' ].split( ',' )[ index ])
if installave > threshold:
main.log.alarm( "{}-node install avg: {} ms > {} ms".format( main.Cluster.numCtrls,
installave, threshold ) )
# Check withdrawal results
if main.flowObj:
threshold = float( main.params[ 'ALARM' ][ 'maxWithdrawLatFlowObj' ].split( ',' )[ index ])
else:
threshold = float( main.params[ 'ALARM' ][ 'maxWithdrawLat' ].split( ',' )[ index ])
if withdrawave > threshold:
main.log.alarm( "{}-node withdraw avg: {} ms > {} ms".format( main.Cluster.numCtrls,
withdrawave, threshold ) )
# write result to database file
if not ( numpy.isnan( installave ) or numpy.isnan( installstd ) or
numpy.isnan( withdrawstd ) or numpy.isnan( withdrawave ) ):
databaseString = "'" + main.commit + "',"
databaseString += str( main.Cluster.numCtrls ) + ","
databaseString += str( batchSize ) + ","
databaseString += str( installave ) + ","
databaseString += str( installstd ) + ","
databaseString += str( withdrawave ) + ","
databaseString += str( withdrawstd ) + "\n"
resultsDB = open( main.dbFileName, "a" )
resultsDB.write( databaseString )
resultsDB.close()
else:
testResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
onpass="Installing and withdrawing intents properly",
onfail="There was something wrong installing and withdrawing intents" )