blob: 8b798bd51e2c7cf587c30edf837e1ef007e498ef [file] [log] [blame]
YPZhangb34b7e12016-06-14 14:28:19 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2015 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070021"""
YPZhangb34b7e12016-06-14 14:28:19 -070022SCPFintentInstallWithdrawLat:
23 - Test the latency of intent installed and withdrawn
24 - Use Push-test-intents command to push intents
25 - Use Null provider with 7 devices and linear topology
26 - Always push intents between 1/6 and 7/5
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070027 - The batch size is defined in parm file. ( default 1,100,1000 )
cameron@onlab.us41c16f52015-07-08 15:40:28 -070028
YPZhangb34b7e12016-06-14 14:28:19 -070029 yunpeng@onlab.us
30"""
cameron@onlab.us41c16f52015-07-08 15:40:28 -070031class SCPFintentInstallWithdrawLat:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070032
cameron@onlab.us41c16f52015-07-08 15:40:28 -070033 def __init__( self ):
34 self.default = ''
35
YPZhangb34b7e12016-06-14 14:28:19 -070036 def CASE0( self, main ):
Devin Lime6fe3c42017-10-18 16:28:40 -070037 import imp
38 import os
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070039 """
YPZhangb34b7e12016-06-14 14:28:19 -070040 - GIT
41 - BUILDING ONOS
42 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
43 This step is usually skipped. Because in a Jenkins driven automated
44 test env. We want Jenkins jobs to pull&build for flexibility to handle
45 different versions of ONOS.
46 - Construct tests variables
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070047 """
Devin Lim58046fa2017-07-05 16:55:00 -070048 try:
49 from tests.dependencies.ONOSSetup import ONOSSetup
50 main.testSetUp = ONOSSetup()
51 except ImportError:
52 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070053 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070054 main.testSetUp.envSetupDescription()
55 stepResult = main.FALSE
56 try:
YPZhangb34b7e12016-06-14 14:28:19 -070057
Devin Lim58046fa2017-07-05 16:55:00 -070058 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
59 main.BENCHUser = main.params[ 'BENCH' ][ 'user' ]
60 main.BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
61 main.MN1Ip = main.params[ 'MN' ][ 'ip1' ]
62 main.maxNodes = int( main.params[ 'max' ] )
63 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
64 main.scale = ( main.params[ 'SCALE' ] ).split( "," )
65 main.timeout = int( main.params[ 'SLEEP' ][ 'timeout' ] )
66 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
67 main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] )
68 main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] )
Devin Lim142b5342017-07-20 15:22:39 -070069 main.intentManagerCfg = main.params[ 'CFG' ][ 'intentManager' ]
70 main.intentConfigRegiCfg = main.params[ 'CFG' ][ 'intentConfigRegi' ]
71 main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ]
Devin Lim58046fa2017-07-05 16:55:00 -070072 main.verifyAttempts = int( main.params[ 'ATTEMPTS' ][ 'verify' ] )
Devin Lime6fe3c42017-10-18 16:28:40 -070073 main.cfgRetry = int( main.params[ 'ATTEMPTS' ][ 'cfg' ] )
74 main.maxInvalidRun = int( main.params[ 'ATTEMPTS' ][ 'maxInvalidRun' ] )
Devin Lim58046fa2017-07-05 16:55:00 -070075 main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
76 main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
77 main.intentsList = ( main.params[ 'TEST' ][ 'intents' ] ).split( "," )
Devin Lime6fe3c42017-10-18 16:28:40 -070078 main.deviceCount = int( main.params[ 'TEST' ][ 'deviceCount' ] )
Devin Lim58046fa2017-07-05 16:55:00 -070079 main.ingress = main.params[ 'TEST' ][ 'ingress' ]
80 main.egress = main.params[ 'TEST' ][ 'egress' ]
81 main.debug = main.params[ 'TEST' ][ 'debug' ]
82 main.flowObj = main.params[ 'TEST' ][ 'flowObj' ]
YPZhangb34b7e12016-06-14 14:28:19 -070083
Devin Lim58046fa2017-07-05 16:55:00 -070084 if main.flowObj == "True":
85 main.flowObj = True
86 main.dbFileName = main.params[ 'DATABASE' ][ 'dbFlowObj' ]
87 else:
88 main.flowObj = False
89 main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ]
YPZhangb34b7e12016-06-14 14:28:19 -070090
Devin Lim58046fa2017-07-05 16:55:00 -070091 for i in range( 0, len( main.intentsList ) ):
92 main.intentsList[ i ] = int( main.intentsList[ i ] )
YPZhangb34b7e12016-06-14 14:28:19 -070093
Devin Lim142b5342017-07-20 15:22:39 -070094 stepResult = main.testSetUp.envSetup()
Devin Lim58046fa2017-07-05 16:55:00 -070095 # Create DataBase file
96 main.log.info( "Create Database file " + main.dbFileName )
97 resultsDB = open( main.dbFileName, "w+" )
98 resultsDB.close()
Devin Lime6fe3c42017-10-18 16:28:40 -070099 file1 = main.params[ "DEPENDENCY" ][ "FILE1" ]
100 main.dependencyPath = os.path.dirname( os.getcwd() ) + main.params[ "DEPENDENCY" ][ "PATH" ]
101 main.intentFuncs = imp.load_source( file1, main.dependencyPath + file1 + ".py" )
Devin Lim58046fa2017-07-05 16:55:00 -0700102 except Exception as e:
103 main.testSetUp.envSetupException( e )
104 main.testSetUp.evnSetupConclusion( stepResult )
105 main.commit = main.commit.split( " " )[ 1 ]
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700106
Jon Hall4ba53f02015-07-29 13:07:41 -0700107 def CASE1( self, main ):
YPZhangb34b7e12016-06-14 14:28:19 -0700108 # Clean up test environment and set up
Jon Hall4ba53f02015-07-29 13:07:41 -0700109 import time
Devin Lim58046fa2017-07-05 16:55:00 -0700110
YPZhangb34b7e12016-06-14 14:28:19 -0700111 main.maxNumBatch = 0
You Wanga0f6ff62018-01-11 15:46:30 -0800112 main.testSetUp.ONOSSetUp( main.Cluster, True,
Devin Lim142b5342017-07-20 15:22:39 -0700113 cellName=main.cellName, killRemoveMax=False )
Devin Lime6fe3c42017-10-18 16:28:40 -0700114 configRetry = 0
115 main.cfgCheck = False
116 while configRetry < main.cfgRetry:
117 # configure apps
118 stepResult = main.TRUE
119 stepResult = stepResult and \
120 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
121 "deviceCount", value=main.deviceCount )
122 stepResult = stepResult and \
123 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
124 "topoShape", value="linear" )
125 stepResult = stepResult and \
126 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
127 "enabled", value="true" )
128 stepResult = stepResult and \
129 main.Cluster.active( 0 ).CLI.setCfg( main.intentManagerCfg,
130 "skipReleaseResourcesOnWithdrawal",
131 value="true" )
132 if main.flowObj:
133 stepResult = stepResult and \
134 main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
135 "useFlowObjectives", value="true" )
Devin Lime6fe3c42017-10-18 16:28:40 -0700136 time.sleep( main.startUpSleep )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700137
Devin Lime6fe3c42017-10-18 16:28:40 -0700138 # balanceMasters
139 stepResult = stepResult and \
140 main.Cluster.active( 0 ).CLI.balanceMasters()
141 if stepResult:
142 main.cfgCheck = True
143 break
144 configRetry += 1
145 time.sleep( main.verifySleep )
Jon Hall4ba53f02015-07-29 13:07:41 -0700146
Devin Lim58046fa2017-07-05 16:55:00 -0700147 time.sleep( main.startUpSleep )
Devin Lime6fe3c42017-10-18 16:28:40 -0700148 if not main.cfgCheck:
149 main.log.error( "Setting configuration to the ONOS failed. Skip the rest of the steps" )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700150
151 def CASE2( self, main ):
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700152 import time
153 import numpy
YPZhangb34b7e12016-06-14 14:28:19 -0700154 import json
Devin Lime6fe3c42017-10-18 16:28:40 -0700155 testResult = main.TRUE
156 main.case( "Installing/Withdrawing intents start" )
157 main.step( "Checking starts" )
158 if main.cfgCheck:
159 print( main.intentsList )
160 for batchSize in main.intentsList:
161 main.log.report( "Intent Batch size: {}".format( batchSize ) )
162 main.batchSize = batchSize
163 main.installLatList = []
164 main.withdrawLatList = []
165 main.validrun = 0
166 main.invalidrun = 0
167 # we use two variables to control the iteration
168 while main.validrun <= main.warmUp + main.sampleSize and main.invalidrun <= main.maxInvalidRun:
169 if main.validrun >= main.warmUp:
170 main.log.info( "================================================" )
171 main.log.info( "Starting test iteration " + str( main.validrun - main.warmUp ) )
172 main.log.info( "Total test iteration: " + str( main.invalidrun + main.validrun ) )
173 main.log.info( "================================================" )
174 else:
175 main.log.info( "====================Warm Up=====================" )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700176
Devin Lime6fe3c42017-10-18 16:28:40 -0700177 # push intents
178 installResult = main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
179 main.egress,
180 batchSize,
181 offset=1,
182 options="-i",
183 timeout=main.timeout,
184 getResponse=True )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700185
Devin Lime6fe3c42017-10-18 16:28:40 -0700186 time.sleep( 2 )
187 main.intentFuncs.sanityCheck( main,
188 ( main.deviceCount - 1 ) * 2,
189 batchSize * main.deviceCount,
190 main.batchSize )
191 if not main.verify:
192 main.log.warn( "Sanity check failed, skipping this iteration..." )
193 continue
194 if isinstance( installResult, str ):
195 if "Failure" in installResult:
196 main.log.error( "Install Intents failure, ignore this iteration." )
197 if main.validrun < main.warmUp:
198 main.validrun += 1
199 continue
200 else:
201 main.invalidrun += 1
202 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700203
Devin Lime6fe3c42017-10-18 16:28:40 -0700204 try:
205 latency = int( installResult.split()[ 5 ] )
206 main.log.info( installResult )
207 except:
208 main.log.error( "Failed to get latency, ignore this iteration." )
209 main.log.error( "Response from ONOS:" )
210 print( installResult )
211 if main.validrun < main.warmUp:
212 main.validrun += 1
213 continue
214 else:
215 main.invalidrun += 1
216 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700217
Devin Lime6fe3c42017-10-18 16:28:40 -0700218 if main.validrun >= main.warmUp:
219 main.installLatList.append( latency )
220 else:
221 main.invalidrun += 1
222 continue
223 # Withdraw Intents
224 withdrawResult = main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
225 main.egress,
226 batchSize,
227 offset=1,
228 options="-w",
229 timeout=main.timeout,
230 getResponse=True )
231 time.sleep( 5 )
232 main.Cluster.active( 0 ).CLI.purgeWithdrawnIntents()
233 main.intentFuncs.sanityCheck( main, ( main.deviceCount - 1 ) * 2, 0, 0 )
234 if not main.verify:
235 main.log.warn( "Sanity check failed, skipping this iteration..." )
236 continue
237 if isinstance( withdrawResult, str ):
238 if "Failure" in withdrawResult:
239 main.log.error( "withdraw Intents failure, ignore this iteration." )
240 if main.validrun < main.warmUp:
241 main.validrun += 1
242 continue
243 else:
244 main.invalidrun += 1
245 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700246
Devin Lime6fe3c42017-10-18 16:28:40 -0700247 try:
248 latency = int( withdrawResult.split()[ 5 ] )
249 main.log.info( withdrawResult )
250 except:
251 main.log.error( "Failed to get latency, ignore this iteration." )
252 main.log.error( "Response from ONOS:" )
253 print( withdrawResult )
254 if main.validrun < main.warmUp:
255 main.validrun += 1
256 continue
257 else:
258 main.invalidrun += 1
259 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700260
Devin Lime6fe3c42017-10-18 16:28:40 -0700261 if main.validrun >= main.warmUp:
262 main.withdrawLatList.append( latency )
263 else:
264 main.invalidrun += 1
265 continue
266 main.validrun += 1
267 result = ( main.TRUE if main.invalidrun <= main.maxInvalidRun else main.FALSE )
268 installave = numpy.average( main.installLatList ) if main.installLatList and result else 0
269 installstd = numpy.std( main.installLatList ) if main.installLatList and result else 0
270 withdrawave = numpy.average( main.withdrawLatList ) if main.withdrawLatList and result else 0
271 withdrawstd = numpy.std( main.withdrawLatList ) if main.withdrawLatList and result else 0
272 testResult = testResult and result
273 # log report
274 main.log.report( "----------------------------------------------------" )
275 main.log.report( "Scale: " + str( main.Cluster.numCtrls ) )
276 main.log.report( "Intent batch: " + str( batchSize ) )
277 main.log.report( "Install average: {} std: {}".format( installave, installstd ) )
278 main.log.report( "Withdraw average: {} std: {}".format( withdrawave, withdrawstd ) )
279 # write result to database file
280 if not ( numpy.isnan( installave ) or numpy.isnan( installstd ) or
281 numpy.isnan( withdrawstd ) or numpy.isnan( withdrawave ) ):
282 databaseString = "'" + main.commit + "',"
283 databaseString += str( main.Cluster.numCtrls ) + ","
284 databaseString += str( batchSize ) + ","
285 databaseString += str( installave ) + ","
286 databaseString += str( installstd ) + ","
287 databaseString += str( withdrawave ) + ","
288 databaseString += str( withdrawstd ) + "\n"
289 resultsDB = open( main.dbFileName, "a" )
290 resultsDB.write( databaseString )
291 resultsDB.close()
292 else:
293 testResult = main.FALSE
294 utilities.assert_equals( expect=main.TRUE,
295 actual=testResult,
296 onpass="Installing and withdrawing intents properly",
297 onfail="There was something wrong installing and withdrawing intents" )