| """ |
| Copyright 2015 Open Networking Foundation ( ONF ) |
| |
| Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>, |
| the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>, |
| or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg> |
| |
| TestON is free software: you can redistribute it and/or modify |
| it under the terms of the GNU General Public License as published by |
| the Free Software Foundation, either version 2 of the License, or |
| ( at your option ) any later version. |
| |
| TestON is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| GNU General Public License for more details. |
| |
| You should have received a copy of the GNU General Public License |
| along with TestON. If not, see <http://www.gnu.org/licenses/>. |
| """ |
| # SCPFintentRerouteLat |
| """ |
| SCPFintentRerouteLat |
| - Test Intent Reroute Latency |
| - Test Algorithm: |
| 1. Start Null Provider reroute Topology |
| 2. Using Push-test-intents to push batch size intents from switch 1 to switch 7 |
| 3. Cut the link between switch 3 and switch 4 ( the path will reroute to switch 8 ) |
| 4. Get the topology time stamp |
| 5. Get Intent reroute( Installed ) time stamp from each nodes |
| 6. Use the latest intent time stamp subtract topology time stamp |
| - This test will run 5 warm up by default, warm up iteration can be setup in Param file |
| - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file |
| - The unit of the latency result is milliseconds |
| """ |
| class SCPFintentRerouteLat: |
| |
| def __init__( self ): |
| self.default = '' |
| |
| def CASE0( self, main ): |
| import imp |
| import os |
| """ |
| - GIT |
| - BUILDING ONOS |
| Pull specific ONOS branch, then Build ONOS ono ONOS Bench. |
| This step is usually skipped. Because in a Jenkins driven automated |
| test env. We want Jenkins jobs to pull&build for flexibility to handle |
| different versions of ONOS. |
| - Construct tests variables |
| """ |
| try: |
| from tests.dependencies.ONOSSetup import ONOSSetup |
| main.testSetUp = ONOSSetup() |
| except ImportError: |
| main.log.error( "ONOSSetup not found. exiting the test" ) |
| main.cleanAndExit() |
| main.testSetUp.envSetupDescription() |
| stepResult = main.FALSE |
| try: |
| main.apps = main.params[ 'ENV' ][ 'cellApps' ] |
| main.BENCHUser = main.params[ 'BENCH' ][ 'user' ] |
| main.BENCHIp = main.params[ 'BENCH' ][ 'ip1' ] |
| main.MN1Ip = main.params[ 'MN' ][ 'ip1' ] |
| main.cellName = main.params[ 'ENV' ][ 'cellName' ] |
| main.scale = ( main.params[ 'SCALE' ] ).split( "," ) |
| main.timeout = int( main.params[ 'SLEEP' ][ 'timeout' ] ) |
| main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] ) |
| main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] ) |
| main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] ) |
| main.setMasterSleep = int( main.params[ 'SLEEP' ][ 'setmaster' ] ) |
| main.verifyAttempts = int( main.params[ 'ATTEMPTS' ][ 'verify' ] ) |
| main.maxInvalidRun = int( main.params[ 'ATTEMPTS' ][ 'maxInvalidRun' ] ) |
| main.cfgRetry = int( main.params[ 'ATTEMPTS' ][ 'cfg' ] ) |
| main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] ) |
| main.intentManagerCfg = main.params[ 'CFG' ][ 'intentManager' ] |
| main.intentConfigRegiCfg = main.params[ 'CFG' ][ 'intentConfigRegi' ] |
| main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ] |
| main.linkCollectionIntentCfg = main.params[ 'CFG' ][ 'linkCollectionIntent' ] |
| main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] ) |
| main.ingress = main.params[ 'TEST' ][ 'ingress' ] |
| main.egress = main.params[ 'TEST' ][ 'egress' ] |
| main.debug = main.params[ 'TEST' ][ 'debug' ] |
| main.flowObj = main.params[ 'TEST' ][ 'flowObj' ] |
| main.deviceCount = int( main.params[ 'TEST' ][ 'deviceCount' ] ) |
| main.end1 = main.params[ 'TEST' ][ 'end1' ] |
| main.end2 = main.params[ 'TEST' ][ 'end2' ] |
| main.searchTerm = main.params[ 'SEARCHTERM' ] |
| if main.flowObj == "True": |
| main.flowObj = True |
| main.dbFileName = main.params[ 'DATABASE' ][ 'dbFlowObj' ] |
| main.intentsList = ( main.params[ 'TEST' ][ 'FObjintents' ] ).split( "," ) |
| else: |
| main.flowObj = False |
| main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ] |
| main.intentsList = ( main.params[ 'TEST' ][ 'intents' ] ).split( "," ) |
| |
| stepResult = main.testSetUp.envSetup() |
| |
| for i in range( 0, len( main.intentsList ) ): |
| main.intentsList[ i ] = int( main.intentsList[ i ] ) |
| # Create DataBase file |
| main.log.info( "Create Database file " + main.dbFileName ) |
| resultsDB = open( main.dbFileName, "w+" ) |
| resultsDB.close() |
| file1 = main.params[ "DEPENDENCY" ][ "FILE1" ] |
| main.dependencyPath = os.path.dirname( os.getcwd() ) + main.params[ "DEPENDENCY" ][ "PATH" ] |
| main.intentRerouteLatFuncs = imp.load_source( file1, main.dependencyPath + file1 + ".py" ) |
| |
| main.record = 0 |
| except Exception as e: |
| main.testSetUp.envSetupException( e ) |
| main.testSetUp.evnSetupConclusion( stepResult ) |
| main.commit = main.commit.split( " " )[ 1 ] |
| |
| def CASE1( self, main ): |
| """ |
| clean up test environment and set up |
| """ |
| import time |
| |
| main.maxNumBatch = 0 |
| main.testSetUp.ONOSSetUp( main.Cluster, True, |
| cellName=main.cellName, killRemoveMax=False ) |
| configRetry = 0 |
| main.cfgCheck = False |
| while configRetry < main.cfgRetry: |
| # configure apps |
| stepResult = main.TRUE |
| stepResult = stepResult and \ |
| main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg, |
| "deviceCount", |
| value=main.deviceCount ) |
| |
| stepResult = stepResult and \ |
| main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg, |
| "topoShape", |
| value="reroute" ) |
| stepResult = stepResult and \ |
| main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg, |
| "enabled", |
| value="true" ) |
| |
| stepResult = stepResult and \ |
| main.Cluster.active( 0 ).CLI.setCfg( main.intentManagerCfg, |
| "skipReleaseResourcesOnWithdrawal", |
| value="true" ) |
| if main.flowObj: |
| stepResult = stepResult and \ |
| main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg, |
| "useFlowObjectives", |
| value="true" ) |
| stepResult = stepResult and \ |
| main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg, |
| "defaultFlowObjectiveCompiler", |
| value=main.linkCollectionIntentCfg ) |
| if stepResult: |
| main.cfgCheck = True |
| break |
| configRetry += 1 |
| time.sleep( main.verifySleep ) |
| |
| time.sleep( main.startUpSleep ) |
| for ctrl in main.Cluster.active(): |
| ctrl.CLI.logSet( "DEBUG", "org.onosproject.metrics.topology" ) |
| ctrl.CLI.logSet( "DEBUG", "org.onosproject.metrics.intent" ) |
| # Balance Master |
| main.Cluster.active( 0 ).CLI.balanceMasters() |
| time.sleep( main.setMasterSleep ) |
| if main.Cluster.numCtrls: |
| main.Cluster.active( 0 ).CLI.deviceRole( main.end1[ 'name' ], main.Cluster.active( 0 ).ipAddress ) |
| main.Cluster.active( 0 ).CLI.deviceRole( main.end2[ 'name' ], main.Cluster.active( 0 ).ipAddress ) |
| time.sleep( main.setMasterSleep ) |
| if not main.cfgCheck: |
| main.log.error( "Setting configuration to the ONOS failed. Skip the rest of the steps" ) |
| |
| def CASE2( self, main ): |
| import time |
| import numpy |
| import datetime |
| import json |
| # from scipy import stats |
| testResult = main.TRUE |
| main.case( "Intent Reroute starts" ) |
| main.step( "Checking intent reroute" ) |
| if main.cfgCheck: |
| print( main.intentsList ) |
| for batchSize in main.intentsList: |
| main.batchSize = batchSize |
| main.log.report( "Intent Batch size: " + str( batchSize ) + "\n " ) |
| firstLocalLatencies = [] |
| lastLocalLatencies = [] |
| firstGlobalLatencies = [] |
| lastGlobalLatencies = [] |
| main.startLine = {} |
| main.validRun = 0 |
| main.invalidRun = 0 |
| while main.validRun <= main.warmUp + main.sampleSize and main.invalidRun <= main.maxInvalidRun: |
| if main.validRun >= main.warmUp: |
| main.log.info( "================================================" ) |
| main.log.info( "Valid iteration: {} ".format( main.validRun - main.warmUp ) ) |
| main.log.info( "Total iteration: {}".format( main.validRun + main.invalidRun ) ) |
| main.log.info( "================================================" ) |
| else: |
| main.log.info( "====================Warm Up=====================" ) |
| |
| # push intents |
| main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress, |
| main.egress, |
| main.batchSize, |
| offset=1, |
| options="-i", |
| timeout=main.timeout ) |
| |
| # check links, flows and intents |
| main.intentRerouteLatFuncs.sanityCheck( main, |
| main.deviceCount * 2, |
| batchSize * ( main.deviceCount - 1 ), |
| main.batchSize ) |
| if not main.verify: |
| main.log.warn( "Sanity check failed, skipping this iteration..." ) |
| continue |
| |
| # Insert one line in karaf.log before link down |
| main.Cluster.command( "log", |
| args=[ "\'Scale: {}, Batch:{}, Iteration: {}\'".format( |
| main.Cluster.numCtrls, batchSize, main.validRun + main.invalidRun ) ], |
| returnBool=True, specificDriver=2 ) |
| # bring link down |
| main.Cluster.active( 0 ).CLI.link( main.end1[ 'port' ], main.end2[ 'port' ], "down", |
| timeout=main.timeout, showResponse=False ) |
| |
| # check links, flows and intents |
| main.intentRerouteLatFuncs.sanityCheck( main, |
| ( main.deviceCount - 1 ) * 2, |
| batchSize * main.deviceCount, |
| main.batchSize ) |
| if not main.verify: |
| main.log.warn( "Sanity check failed, skipping this iteration..." ) |
| continue |
| |
| # Get timestamp of last LINK_REMOVED event as separator between iterations |
| skip = False |
| for i in range( main.Cluster.numCtrls ): |
| logNum = main.intentRerouteLatFuncs.getLogNum( main, i ) |
| timestamp = str( main.Cluster.active( i ).CLI.getTimeStampFromLog( "last", |
| "LINK_REMOVED", |
| "time = ", " ", |
| logNum=logNum ) ) |
| if timestamp == main.ERROR: |
| # Try again in case that the log number just increased |
| logNum = main.intentRerouteLatFuncs.getLogNum( main, i ) |
| timestamp = str( main.Cluster.active( i ).CLI.getTimeStampFromLog( "last", |
| "LINK_REMOVED", |
| "time = ", " ", |
| logNum=logNum ) ) |
| if timestamp == main.ERROR: |
| main.log.warn( "Cannot find the event we want in the log, skipping this iteration..." ) |
| main.intentRerouteLatFuncs.bringBackTopology( main ) |
| if main.validRun >= main.warmUp: |
| main.invalidRun += 1 |
| else: |
| main.validRun += 1 |
| skip = True |
| break |
| else: |
| main.startLine[ i ] = timestamp |
| main.log.info( "Timestamp of last LINK_REMOVED event on node {} is {}".format( i + 1, |
| main.startLine[ i ] ) ) |
| if skip: |
| continue |
| |
| # calculate values |
| topologyTimestamps = main.intentRerouteLatFuncs.getTopologyTimestamps( main ) |
| intentTimestamps = main.intentRerouteLatFuncs.getIntentTimestamps( main ) |
| if intentTimestamps == main.ERROR or topologyTimestamps == main.ERROR: |
| main.log.info( "Got invalid timestamp, skipping this iteration..." ) |
| main.intentRerouteLatFuncs.bringBackTopology( main ) |
| if main.validRun >= main.warmUp: |
| main.invalidRun += 1 |
| else: |
| main.validRun += 1 |
| continue |
| else: |
| main.log.info( "Got valid timestamps" ) |
| |
| firstLocalLatnecy, lastLocalLatnecy, firstGlobalLatency, lastGlobalLatnecy \ |
| = main.intentRerouteLatFuncs.calculateLatency( main, topologyTimestamps, intentTimestamps ) |
| if firstLocalLatnecy < 0: |
| main.log.info( "Got negative latency, skipping this iteration..." ) |
| main.intentRerouteLatFuncs.bringBackTopology( main ) |
| if main.validRun >= main.warmUp: |
| main.invalidRun += 1 |
| else: |
| main.validRun += 1 |
| continue |
| else: |
| main.log.info( "Got valid latencies" ) |
| main.validRun += 1 |
| |
| if main.validRun >= main.warmUp: |
| firstLocalLatencies.append( firstLocalLatnecy ) |
| lastLocalLatencies.append( lastLocalLatnecy ) |
| firstGlobalLatencies.append( firstGlobalLatency ) |
| lastGlobalLatencies.append( lastGlobalLatnecy ) |
| |
| # bring up link and withdraw intents |
| main.Cluster.active( 0 ).CLI.link( main.end1[ 'port' ], |
| main.end2[ 'port' ], |
| "up", |
| timeout=main.timeout ) |
| main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress, |
| main.egress, |
| batchSize, |
| offset=1, |
| options="-w", |
| timeout=main.timeout ) |
| main.Cluster.active( 0 ).CLI.purgeWithdrawnIntents() |
| |
| # check links, flows and intents |
| main.intentRerouteLatFuncs.sanityCheck( main, main.deviceCount * 2, 0, 0 ) |
| if not main.verify: |
| main.log.warn( "Sanity check failed, skipping this iteration..." ) |
| continue |
| result = ( main.TRUE if main.invalidRun <= main.maxInvalidRun else main.FALSE ) |
| aveLocalLatency = numpy.average( lastLocalLatencies ) if lastLocalLatencies and result else 0 |
| aveGlobalLatency = numpy.average( lastGlobalLatencies ) if lastGlobalLatencies and result else 0 |
| stdLocalLatency = numpy.std( lastLocalLatencies ) if lastLocalLatencies and result else 0 |
| stdGlobalLatency = numpy.std( lastGlobalLatencies ) if lastGlobalLatencies and result else 0 |
| testResult = testResult and result |
| |
| main.log.report( "Scale: " + str( main.Cluster.numCtrls ) + " \tIntent batch: " + str( batchSize ) ) |
| main.log.report( "Local latency average:................" + str( aveLocalLatency ) ) |
| main.log.report( "Global latency average:................" + str( aveGlobalLatency ) ) |
| main.log.report( "Local latency std:................" + str( stdLocalLatency ) ) |
| main.log.report( "Global latency std:................" + str( stdGlobalLatency ) ) |
| main.log.report( "________________________________________________________" ) |
| |
| if not ( numpy.isnan( aveLocalLatency ) or numpy.isnan( aveGlobalLatency ) ): |
| # check if got NaN for result |
| resultsDB = open( main.dbFileName, "a" ) |
| resultsDB.write( "'" + main.commit + "'," ) |
| resultsDB.write( str( main.Cluster.numCtrls ) + "," ) |
| resultsDB.write( str( batchSize ) + "," ) |
| resultsDB.write( str( aveLocalLatency ) + "," ) |
| resultsDB.write( str( stdLocalLatency ) + "\n" ) |
| resultsDB.close() |
| else: |
| testResult = main.FALSE |
| utilities.assert_equals( expect=main.TRUE, |
| actual=testResult, |
| onpass="Installing and withdrawing intents reroute properly", |
| onfail="There was something wrong installing and withdrawing intents reroute" ) |