YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 1 | """ |
Jeremy Ronquillo | b27ce4c | 2017-07-17 12:41:28 -0700 | [diff] [blame] | 2 | Copyright 2015 Open Networking Foundation (ONF) |
| 3 | |
| 4 | Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>, |
| 5 | the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>, |
| 6 | or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg> |
| 7 | |
| 8 | TestON is free software: you can redistribute it and/or modify |
| 9 | it under the terms of the GNU General Public License as published by |
| 10 | the Free Software Foundation, either version 2 of the License, or |
| 11 | (at your option) any later version. |
| 12 | |
| 13 | TestON is distributed in the hope that it will be useful, |
| 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | GNU General Public License for more details. |
| 17 | |
| 18 | You should have received a copy of the GNU General Public License |
| 19 | along with TestON. If not, see <http://www.gnu.org/licenses/>. |
| 20 | """ |
| 21 | |
| 22 | """ |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 23 | SCPFintentInstallWithdrawLat: |
| 24 | - Test the latency of intent installed and withdrawn |
| 25 | - Use Push-test-intents command to push intents |
| 26 | - Use Null provider with 7 devices and linear topology |
| 27 | - Always push intents between 1/6 and 7/5 |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 28 | - The batch size is defined in parm file. ( default 1,100,1000) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 29 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 30 | yunpeng@onlab.us |
| 31 | """ |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 32 | class SCPFintentInstallWithdrawLat: |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 33 | def __init__( self ): |
| 34 | self.default = '' |
| 35 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 36 | def CASE0( self, main ): |
| 37 | ''' |
| 38 | - GIT |
| 39 | - BUILDING ONOS |
| 40 | Pull specific ONOS branch, then Build ONOS ono ONOS Bench. |
| 41 | This step is usually skipped. Because in a Jenkins driven automated |
| 42 | test env. We want Jenkins jobs to pull&build for flexibility to handle |
| 43 | different versions of ONOS. |
| 44 | - Construct tests variables |
| 45 | ''' |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 46 | |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 47 | try: |
| 48 | from tests.dependencies.ONOSSetup import ONOSSetup |
| 49 | main.testSetUp = ONOSSetup() |
| 50 | except ImportError: |
| 51 | main.log.error( "ONOSSetup not found. exiting the test" ) |
| 52 | main.exit() |
| 53 | main.testSetUp.envSetupDescription() |
| 54 | stepResult = main.FALSE |
| 55 | try: |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 56 | |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 57 | main.apps = main.params[ 'ENV' ][ 'cellApps' ] |
| 58 | main.BENCHUser = main.params[ 'BENCH' ][ 'user' ] |
| 59 | main.BENCHIp = main.params[ 'BENCH' ][ 'ip1' ] |
| 60 | main.MN1Ip = main.params[ 'MN' ][ 'ip1' ] |
| 61 | main.maxNodes = int( main.params[ 'max' ] ) |
| 62 | main.cellName = main.params[ 'ENV' ][ 'cellName' ] |
| 63 | main.scale = ( main.params[ 'SCALE' ] ).split( "," ) |
| 64 | main.timeout = int( main.params[ 'SLEEP' ][ 'timeout' ] ) |
| 65 | main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] ) |
| 66 | main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] ) |
| 67 | main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] ) |
| 68 | main.verifyAttempts = int( main.params[ 'ATTEMPTS' ][ 'verify' ] ) |
| 69 | main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] ) |
| 70 | main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] ) |
| 71 | main.intentsList = ( main.params[ 'TEST' ][ 'intents' ] ).split( "," ) |
| 72 | main.ingress = main.params[ 'TEST' ][ 'ingress' ] |
| 73 | main.egress = main.params[ 'TEST' ][ 'egress' ] |
| 74 | main.debug = main.params[ 'TEST' ][ 'debug' ] |
| 75 | main.flowObj = main.params[ 'TEST' ][ 'flowObj' ] |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 76 | |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 77 | if main.flowObj == "True": |
| 78 | main.flowObj = True |
| 79 | main.dbFileName = main.params[ 'DATABASE' ][ 'dbFlowObj' ] |
| 80 | else: |
| 81 | main.flowObj = False |
| 82 | main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ] |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 83 | |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 84 | for i in range( 0, len( main.intentsList ) ): |
| 85 | main.intentsList[ i ] = int( main.intentsList[ i ] ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 86 | |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 87 | stepResult = main.testSetUp.gitPulling() |
| 88 | # Create DataBase file |
| 89 | main.log.info( "Create Database file " + main.dbFileName ) |
| 90 | resultsDB = open( main.dbFileName, "w+" ) |
| 91 | resultsDB.close() |
| 92 | except Exception as e: |
| 93 | main.testSetUp.envSetupException( e ) |
| 94 | main.testSetUp.evnSetupConclusion( stepResult ) |
| 95 | main.commit = main.commit.split( " " )[ 1 ] |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 96 | def CASE1( self, main ): |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 97 | # Clean up test environment and set up |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 98 | import time |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 99 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 100 | main.maxNumBatch = 0 |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 101 | main.testSetUp.getNumCtrls( True ) |
| 102 | main.testSetUp.envSetup( includeGitPull=False, makeMaxNodes=False ) |
| 103 | main.testSetUp.ONOSSetUp( main.MN1Ip, True, |
| 104 | cellName=main.cellName, killRemoveMax=False, |
| 105 | CtrlsSet=False ) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 106 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 107 | # configure apps |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 108 | main.CLIs[ 0 ].setCfg( "org.onosproject.provider.nil.NullProviders", "deviceCount", value=7 ) |
| 109 | main.CLIs[ 0 ].setCfg( "org.onosproject.provider.nil.NullProviders", "topoShape", value="linear" ) |
| 110 | main.CLIs[ 0 ].setCfg( "org.onosproject.provider.nil.NullProviders", "enabled", value="true" ) |
| 111 | main.CLIs[ 0 ].setCfg( "org.onosproject.net.intent.impl.IntentManager", "skipReleaseResourcesOnWithdrawal", value="true" ) |
YPZhang | e6ef82a | 2016-07-05 16:48:15 -0700 | [diff] [blame] | 112 | if main.flowObj: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 113 | main.CLIs[ 0 ].setCfg( "org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator", |
| 114 | "useFlowObjectives", value="true" ) |
| 115 | main.CLIs[ 0 ].setCfg( "org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator", |
You Wang | 106d0fa | 2017-05-15 17:22:15 -0700 | [diff] [blame] | 116 | "defaultFlowObjectiveCompiler", |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 117 | value='org.onosproject.net.intent.impl.compiler.LinkCollectionIntentObjectiveCompiler' ) |
| 118 | time.sleep( main.startUpSleep ) |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 119 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 120 | # balanceMasters |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 121 | main.CLIs[ 0 ].balanceMasters() |
| 122 | time.sleep( main.startUpSleep ) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 123 | |
| 124 | def CASE2( self, main ): |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 125 | import time |
| 126 | import numpy |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 127 | import json |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 128 | print( main.intentsList ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 129 | for batchSize in main.intentsList: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 130 | main.log.report( "Intent Batch size: {}".format( batchSize ) ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 131 | main.installLatList = [] |
| 132 | main.withdrawLatList = [] |
| 133 | validrun = 0 |
| 134 | invalidrun = 0 |
| 135 | # we use two variables to control the iteration |
| 136 | while validrun <= main.warmUp + main.sampleSize and invalidrun < 20: |
| 137 | if validrun >= main.warmUp: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 138 | main.log.info( "================================================" ) |
| 139 | main.log.info( "Starting test iteration " + str( validrun - main.warmUp ) ) |
| 140 | main.log.info( "Total test iteration: " + str( invalidrun + validrun ) ) |
| 141 | main.log.info( "================================================" ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 142 | else: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 143 | main.log.info( "====================Warm Up=====================" ) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 144 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 145 | # push intents |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 146 | installResult = main.CLIs[ 0 ].pushTestIntents( main.ingress, main.egress, batchSize, |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 147 | offset=1, options="-i", timeout=main.timeout, |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 148 | getResponse=True ) |
| 149 | if type( installResult ) is str: |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 150 | if "Failure" in installResult: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 151 | main.log.error( "Install Intents failure, ignore this iteration." ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 152 | if validrun < main.warmUp: |
| 153 | validrun += 1 |
| 154 | continue |
| 155 | else: |
| 156 | invalidrun += 1 |
| 157 | continue |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 158 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 159 | try: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 160 | latency = int( installResult.split()[ 5 ] ) |
| 161 | main.log.info( installResult ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 162 | except: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 163 | main.log.error( "Failed to get latency, ignore this iteration." ) |
| 164 | main.log.error( "Response from ONOS:" ) |
| 165 | print( installResult ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 166 | if validrun < main.warmUp: |
| 167 | validrun += 1 |
| 168 | continue |
| 169 | else: |
| 170 | invalidrun += 1 |
| 171 | continue |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 172 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 173 | if validrun >= main.warmUp: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 174 | main.installLatList.append( latency ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 175 | else: |
| 176 | invalidrun += 1 |
| 177 | continue |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 178 | time.sleep( 2 ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 179 | # Withdraw Intents |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 180 | withdrawResult = main.CLIs[ 0 ].pushTestIntents( main.ingress, main.egress, batchSize, |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 181 | offset=1, options="-w", timeout=main.timeout, |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 182 | getResponse=True ) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 183 | |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 184 | if type( withdrawResult ) is str: |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 185 | if "Failure" in withdrawResult: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 186 | main.log.error( "withdraw Intents failure, ignore this iteration." ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 187 | if validrun < main.warmUp: |
| 188 | validrun += 1 |
| 189 | continue |
| 190 | else: |
| 191 | invalidrun += 1 |
| 192 | continue |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 193 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 194 | try: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 195 | latency = int( withdrawResult.split()[ 5 ] ) |
| 196 | main.log.info( withdrawResult ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 197 | except: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 198 | main.log.error( "Failed to get latency, ignore this iteration." ) |
| 199 | main.log.error( "Response from ONOS:" ) |
| 200 | print( withdrawResult ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 201 | if validrun < main.warmUp: |
| 202 | validrun += 1 |
| 203 | continue |
| 204 | else: |
| 205 | invalidrun += 1 |
| 206 | continue |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 207 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 208 | if validrun >= main.warmUp: |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 209 | main.withdrawLatList.append( latency ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 210 | else: |
| 211 | invalidrun += 1 |
| 212 | continue |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 213 | time.sleep( 2 ) |
| 214 | main.CLIs[ 0 ].purgeWithdrawnIntents() |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 215 | validrun += 1 |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 216 | installave = numpy.average( main.installLatList ) |
| 217 | installstd = numpy.std( main.installLatList ) |
| 218 | withdrawave = numpy.average( main.withdrawLatList ) |
| 219 | withdrawstd = numpy.std( main.withdrawLatList ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 220 | # log report |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 221 | main.log.report( "----------------------------------------------------" ) |
| 222 | main.log.report( "Scale: " + str( main.numCtrls ) ) |
| 223 | main.log.report( "Intent batch: " + str( batchSize ) ) |
| 224 | main.log.report( "Install average: {} std: {}".format( installave, installstd ) ) |
| 225 | main.log.report( "Withdraw average: {} std: {}".format( withdrawave, withdrawstd ) ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 226 | # write result to database file |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 227 | if not ( numpy.isnan( installave ) or numpy.isnan( installstd ) or\ |
| 228 | numpy.isnan( withdrawstd ) or numpy.isnan( withdrawave ) ): |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 229 | databaseString = "'" + main.commit + "'," |
Devin Lim | 58046fa | 2017-07-05 16:55:00 -0700 | [diff] [blame] | 230 | databaseString += str( main.numCtrls ) + "," |
| 231 | databaseString += str( batchSize ) + "," |
| 232 | databaseString += str( installave ) + "," |
| 233 | databaseString += str( installstd ) + "," |
| 234 | databaseString += str( withdrawave ) + "," |
| 235 | databaseString += str( withdrawstd ) + "\n" |
| 236 | resultsDB = open( main.dbFileName, "a" ) |
| 237 | resultsDB.write( databaseString ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 238 | resultsDB.close() |