blob: ef1cfb98dc05bc752a1a54595b4d232f4e761f81 [file] [log] [blame]
YPZhangb34b7e12016-06-14 14:28:19 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2015 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070021"""
YPZhangb34b7e12016-06-14 14:28:19 -070022SCPFintentInstallWithdrawLat:
23 - Test the latency of intent installed and withdrawn
24 - Use Push-test-intents command to push intents
25 - Use Null provider with 7 devices and linear topology
26 - Always push intents between 1/6 and 7/5
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070027 - The batch size is defined in parm file. ( default 1,100,1000 )
cameron@onlab.us41c16f52015-07-08 15:40:28 -070028
YPZhangb34b7e12016-06-14 14:28:19 -070029 yunpeng@onlab.us
30"""
cameron@onlab.us41c16f52015-07-08 15:40:28 -070031class SCPFintentInstallWithdrawLat:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070032
cameron@onlab.us41c16f52015-07-08 15:40:28 -070033 def __init__( self ):
34 self.default = ''
35
YPZhangb34b7e12016-06-14 14:28:19 -070036 def CASE0( self, main ):
Devin Lime6fe3c42017-10-18 16:28:40 -070037 import imp
38 import os
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070039 """
YPZhangb34b7e12016-06-14 14:28:19 -070040 - GIT
41 - BUILDING ONOS
42 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
43 This step is usually skipped. Because in a Jenkins driven automated
44 test env. We want Jenkins jobs to pull&build for flexibility to handle
45 different versions of ONOS.
46 - Construct tests variables
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070047 """
Devin Lim58046fa2017-07-05 16:55:00 -070048 try:
49 from tests.dependencies.ONOSSetup import ONOSSetup
50 main.testSetUp = ONOSSetup()
51 except ImportError:
52 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070053 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070054 main.testSetUp.envSetupDescription()
55 stepResult = main.FALSE
56 try:
YPZhangb34b7e12016-06-14 14:28:19 -070057
Devin Lim58046fa2017-07-05 16:55:00 -070058 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
59 main.BENCHUser = main.params[ 'BENCH' ][ 'user' ]
60 main.BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
61 main.MN1Ip = main.params[ 'MN' ][ 'ip1' ]
62 main.maxNodes = int( main.params[ 'max' ] )
63 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
64 main.scale = ( main.params[ 'SCALE' ] ).split( "," )
65 main.timeout = int( main.params[ 'SLEEP' ][ 'timeout' ] )
66 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
67 main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] )
68 main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] )
Devin Lim142b5342017-07-20 15:22:39 -070069 main.intentManagerCfg = main.params[ 'CFG' ][ 'intentManager' ]
70 main.intentConfigRegiCfg = main.params[ 'CFG' ][ 'intentConfigRegi' ]
71 main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ]
72 main.linkCollectionIntentCfg = main.params[ 'CFG' ][ 'linkCollectionIntent' ]
Devin Lim58046fa2017-07-05 16:55:00 -070073 main.verifyAttempts = int( main.params[ 'ATTEMPTS' ][ 'verify' ] )
Devin Lime6fe3c42017-10-18 16:28:40 -070074 main.cfgRetry = int( main.params[ 'ATTEMPTS' ][ 'cfg' ] )
75 main.maxInvalidRun = int( main.params[ 'ATTEMPTS' ][ 'maxInvalidRun' ] )
Devin Lim58046fa2017-07-05 16:55:00 -070076 main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
77 main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
78 main.intentsList = ( main.params[ 'TEST' ][ 'intents' ] ).split( "," )
Devin Lime6fe3c42017-10-18 16:28:40 -070079 main.deviceCount = int( main.params[ 'TEST' ][ 'deviceCount' ] )
Devin Lim58046fa2017-07-05 16:55:00 -070080 main.ingress = main.params[ 'TEST' ][ 'ingress' ]
81 main.egress = main.params[ 'TEST' ][ 'egress' ]
82 main.debug = main.params[ 'TEST' ][ 'debug' ]
83 main.flowObj = main.params[ 'TEST' ][ 'flowObj' ]
YPZhangb34b7e12016-06-14 14:28:19 -070084
Devin Lim58046fa2017-07-05 16:55:00 -070085 if main.flowObj == "True":
86 main.flowObj = True
87 main.dbFileName = main.params[ 'DATABASE' ][ 'dbFlowObj' ]
88 else:
89 main.flowObj = False
90 main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ]
YPZhangb34b7e12016-06-14 14:28:19 -070091
Devin Lim58046fa2017-07-05 16:55:00 -070092 for i in range( 0, len( main.intentsList ) ):
93 main.intentsList[ i ] = int( main.intentsList[ i ] )
YPZhangb34b7e12016-06-14 14:28:19 -070094
Devin Lim142b5342017-07-20 15:22:39 -070095 stepResult = main.testSetUp.envSetup()
Devin Lim58046fa2017-07-05 16:55:00 -070096 # Create DataBase file
97 main.log.info( "Create Database file " + main.dbFileName )
98 resultsDB = open( main.dbFileName, "w+" )
99 resultsDB.close()
Devin Lime6fe3c42017-10-18 16:28:40 -0700100 file1 = main.params[ "DEPENDENCY" ][ "FILE1" ]
101 main.dependencyPath = os.path.dirname( os.getcwd() ) + main.params[ "DEPENDENCY" ][ "PATH" ]
102 main.intentFuncs = imp.load_source( file1, main.dependencyPath + file1 + ".py" )
Devin Lim58046fa2017-07-05 16:55:00 -0700103 except Exception as e:
104 main.testSetUp.envSetupException( e )
105 main.testSetUp.evnSetupConclusion( stepResult )
106 main.commit = main.commit.split( " " )[ 1 ]
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700107
Jon Hall4ba53f02015-07-29 13:07:41 -0700108 def CASE1( self, main ):
YPZhangb34b7e12016-06-14 14:28:19 -0700109 # Clean up test environment and set up
Jon Hall4ba53f02015-07-29 13:07:41 -0700110 import time
Devin Lim58046fa2017-07-05 16:55:00 -0700111
YPZhangb34b7e12016-06-14 14:28:19 -0700112 main.maxNumBatch = 0
You Wanga0f6ff62018-01-11 15:46:30 -0800113 main.testSetUp.ONOSSetUp( main.Cluster, True,
Devin Lim142b5342017-07-20 15:22:39 -0700114 cellName=main.cellName, killRemoveMax=False )
Devin Lime6fe3c42017-10-18 16:28:40 -0700115 configRetry = 0
116 main.cfgCheck = False
117 while configRetry < main.cfgRetry:
118 # configure apps
119 stepResult = main.TRUE
120 stepResult = stepResult and \
121 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
122 "deviceCount", value=main.deviceCount )
123 stepResult = stepResult and \
124 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
125 "topoShape", value="linear" )
126 stepResult = stepResult and \
127 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
128 "enabled", value="true" )
129 stepResult = stepResult and \
130 main.Cluster.active( 0 ).CLI.setCfg( main.intentManagerCfg,
131 "skipReleaseResourcesOnWithdrawal",
132 value="true" )
133 if main.flowObj:
134 stepResult = stepResult and \
135 main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
136 "useFlowObjectives", value="true" )
137 stepResult = stepResult and \
138 main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
139 "defaultFlowObjectiveCompiler",
140 value=main.linkCollectionIntentCfg )
141 time.sleep( main.startUpSleep )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700142
Devin Lime6fe3c42017-10-18 16:28:40 -0700143 # balanceMasters
144 stepResult = stepResult and \
145 main.Cluster.active( 0 ).CLI.balanceMasters()
146 if stepResult:
147 main.cfgCheck = True
148 break
149 configRetry += 1
150 time.sleep( main.verifySleep )
Jon Hall4ba53f02015-07-29 13:07:41 -0700151
Devin Lim58046fa2017-07-05 16:55:00 -0700152 time.sleep( main.startUpSleep )
Devin Lime6fe3c42017-10-18 16:28:40 -0700153 if not main.cfgCheck:
154 main.log.error( "Setting configuration to the ONOS failed. Skip the rest of the steps" )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700155
156 def CASE2( self, main ):
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700157 import time
158 import numpy
YPZhangb34b7e12016-06-14 14:28:19 -0700159 import json
Devin Lime6fe3c42017-10-18 16:28:40 -0700160 testResult = main.TRUE
161 main.case( "Installing/Withdrawing intents start" )
162 main.step( "Checking starts" )
163 if main.cfgCheck:
164 print( main.intentsList )
165 for batchSize in main.intentsList:
166 main.log.report( "Intent Batch size: {}".format( batchSize ) )
167 main.batchSize = batchSize
168 main.installLatList = []
169 main.withdrawLatList = []
170 main.validrun = 0
171 main.invalidrun = 0
172 # we use two variables to control the iteration
173 while main.validrun <= main.warmUp + main.sampleSize and main.invalidrun <= main.maxInvalidRun:
174 if main.validrun >= main.warmUp:
175 main.log.info( "================================================" )
176 main.log.info( "Starting test iteration " + str( main.validrun - main.warmUp ) )
177 main.log.info( "Total test iteration: " + str( main.invalidrun + main.validrun ) )
178 main.log.info( "================================================" )
179 else:
180 main.log.info( "====================Warm Up=====================" )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700181
Devin Lime6fe3c42017-10-18 16:28:40 -0700182 # push intents
183 installResult = main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
184 main.egress,
185 batchSize,
186 offset=1,
187 options="-i",
188 timeout=main.timeout,
189 getResponse=True )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700190
Devin Lime6fe3c42017-10-18 16:28:40 -0700191 time.sleep( 2 )
192 main.intentFuncs.sanityCheck( main,
193 ( main.deviceCount - 1 ) * 2,
194 batchSize * main.deviceCount,
195 main.batchSize )
196 if not main.verify:
197 main.log.warn( "Sanity check failed, skipping this iteration..." )
198 continue
199 if isinstance( installResult, str ):
200 if "Failure" in installResult:
201 main.log.error( "Install Intents failure, ignore this iteration." )
202 if main.validrun < main.warmUp:
203 main.validrun += 1
204 continue
205 else:
206 main.invalidrun += 1
207 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700208
Devin Lime6fe3c42017-10-18 16:28:40 -0700209 try:
210 latency = int( installResult.split()[ 5 ] )
211 main.log.info( installResult )
212 except:
213 main.log.error( "Failed to get latency, ignore this iteration." )
214 main.log.error( "Response from ONOS:" )
215 print( installResult )
216 if main.validrun < main.warmUp:
217 main.validrun += 1
218 continue
219 else:
220 main.invalidrun += 1
221 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700222
Devin Lime6fe3c42017-10-18 16:28:40 -0700223 if main.validrun >= main.warmUp:
224 main.installLatList.append( latency )
225 else:
226 main.invalidrun += 1
227 continue
228 # Withdraw Intents
229 withdrawResult = main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
230 main.egress,
231 batchSize,
232 offset=1,
233 options="-w",
234 timeout=main.timeout,
235 getResponse=True )
236 time.sleep( 5 )
237 main.Cluster.active( 0 ).CLI.purgeWithdrawnIntents()
238 main.intentFuncs.sanityCheck( main, ( main.deviceCount - 1 ) * 2, 0, 0 )
239 if not main.verify:
240 main.log.warn( "Sanity check failed, skipping this iteration..." )
241 continue
242 if isinstance( withdrawResult, str ):
243 if "Failure" in withdrawResult:
244 main.log.error( "withdraw Intents failure, ignore this iteration." )
245 if main.validrun < main.warmUp:
246 main.validrun += 1
247 continue
248 else:
249 main.invalidrun += 1
250 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700251
Devin Lime6fe3c42017-10-18 16:28:40 -0700252 try:
253 latency = int( withdrawResult.split()[ 5 ] )
254 main.log.info( withdrawResult )
255 except:
256 main.log.error( "Failed to get latency, ignore this iteration." )
257 main.log.error( "Response from ONOS:" )
258 print( withdrawResult )
259 if main.validrun < main.warmUp:
260 main.validrun += 1
261 continue
262 else:
263 main.invalidrun += 1
264 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700265
Devin Lime6fe3c42017-10-18 16:28:40 -0700266 if main.validrun >= main.warmUp:
267 main.withdrawLatList.append( latency )
268 else:
269 main.invalidrun += 1
270 continue
271 main.validrun += 1
272 result = ( main.TRUE if main.invalidrun <= main.maxInvalidRun else main.FALSE )
273 installave = numpy.average( main.installLatList ) if main.installLatList and result else 0
274 installstd = numpy.std( main.installLatList ) if main.installLatList and result else 0
275 withdrawave = numpy.average( main.withdrawLatList ) if main.withdrawLatList and result else 0
276 withdrawstd = numpy.std( main.withdrawLatList ) if main.withdrawLatList and result else 0
277 testResult = testResult and result
278 # log report
279 main.log.report( "----------------------------------------------------" )
280 main.log.report( "Scale: " + str( main.Cluster.numCtrls ) )
281 main.log.report( "Intent batch: " + str( batchSize ) )
282 main.log.report( "Install average: {} std: {}".format( installave, installstd ) )
283 main.log.report( "Withdraw average: {} std: {}".format( withdrawave, withdrawstd ) )
284 # write result to database file
285 if not ( numpy.isnan( installave ) or numpy.isnan( installstd ) or
286 numpy.isnan( withdrawstd ) or numpy.isnan( withdrawave ) ):
287 databaseString = "'" + main.commit + "',"
288 databaseString += str( main.Cluster.numCtrls ) + ","
289 databaseString += str( batchSize ) + ","
290 databaseString += str( installave ) + ","
291 databaseString += str( installstd ) + ","
292 databaseString += str( withdrawave ) + ","
293 databaseString += str( withdrawstd ) + "\n"
294 resultsDB = open( main.dbFileName, "a" )
295 resultsDB.write( databaseString )
296 resultsDB.close()
297 else:
298 testResult = main.FALSE
299 utilities.assert_equals( expect=main.TRUE,
300 actual=testResult,
301 onpass="Installing and withdrawing intents properly",
302 onfail="There was something wrong installing and withdrawing intents" )