blob: 78a2b3443b73ac0ad89ac77e22d6d064dc5d33e3 [file] [log] [blame]
YPZhangb34b7e12016-06-14 14:28:19 -07001"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07002Copyright 2015 Open Networking Foundation (ONF)
3
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
11 (at your option) any later version.
12
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
21
22"""
YPZhangb34b7e12016-06-14 14:28:19 -070023SCPFintentInstallWithdrawLat:
24 - Test the latency of intent installed and withdrawn
25 - Use Push-test-intents command to push intents
26 - Use Null provider with 7 devices and linear topology
27 - Always push intents between 1/6 and 7/5
Devin Lim58046fa2017-07-05 16:55:00 -070028 - The batch size is defined in parm file. ( default 1,100,1000)
cameron@onlab.us41c16f52015-07-08 15:40:28 -070029
YPZhangb34b7e12016-06-14 14:28:19 -070030 yunpeng@onlab.us
31"""
cameron@onlab.us41c16f52015-07-08 15:40:28 -070032class SCPFintentInstallWithdrawLat:
cameron@onlab.us41c16f52015-07-08 15:40:28 -070033 def __init__( self ):
34 self.default = ''
35
YPZhangb34b7e12016-06-14 14:28:19 -070036 def CASE0( self, main ):
37 '''
38 - GIT
39 - BUILDING ONOS
40 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
41 This step is usually skipped. Because in a Jenkins driven automated
42 test env. We want Jenkins jobs to pull&build for flexibility to handle
43 different versions of ONOS.
44 - Construct tests variables
45 '''
YPZhangb34b7e12016-06-14 14:28:19 -070046
Devin Lim58046fa2017-07-05 16:55:00 -070047 try:
48 from tests.dependencies.ONOSSetup import ONOSSetup
49 main.testSetUp = ONOSSetup()
50 except ImportError:
51 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070052 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070053 main.testSetUp.envSetupDescription()
54 stepResult = main.FALSE
55 try:
YPZhangb34b7e12016-06-14 14:28:19 -070056
Devin Lim58046fa2017-07-05 16:55:00 -070057 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
58 main.BENCHUser = main.params[ 'BENCH' ][ 'user' ]
59 main.BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
60 main.MN1Ip = main.params[ 'MN' ][ 'ip1' ]
61 main.maxNodes = int( main.params[ 'max' ] )
62 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
63 main.scale = ( main.params[ 'SCALE' ] ).split( "," )
64 main.timeout = int( main.params[ 'SLEEP' ][ 'timeout' ] )
65 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
66 main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] )
67 main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] )
Devin Lim142b5342017-07-20 15:22:39 -070068 main.intentManagerCfg = main.params[ 'CFG' ][ 'intentManager' ]
69 main.intentConfigRegiCfg = main.params[ 'CFG' ][ 'intentConfigRegi' ]
70 main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ]
71 main.linkCollectionIntentCfg = main.params[ 'CFG' ][ 'linkCollectionIntent' ]
Devin Lim58046fa2017-07-05 16:55:00 -070072 main.verifyAttempts = int( main.params[ 'ATTEMPTS' ][ 'verify' ] )
73 main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
74 main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
75 main.intentsList = ( main.params[ 'TEST' ][ 'intents' ] ).split( "," )
76 main.ingress = main.params[ 'TEST' ][ 'ingress' ]
77 main.egress = main.params[ 'TEST' ][ 'egress' ]
78 main.debug = main.params[ 'TEST' ][ 'debug' ]
79 main.flowObj = main.params[ 'TEST' ][ 'flowObj' ]
YPZhangb34b7e12016-06-14 14:28:19 -070080
Devin Lim58046fa2017-07-05 16:55:00 -070081 if main.flowObj == "True":
82 main.flowObj = True
83 main.dbFileName = main.params[ 'DATABASE' ][ 'dbFlowObj' ]
84 else:
85 main.flowObj = False
86 main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ]
YPZhangb34b7e12016-06-14 14:28:19 -070087
Devin Lim58046fa2017-07-05 16:55:00 -070088 for i in range( 0, len( main.intentsList ) ):
89 main.intentsList[ i ] = int( main.intentsList[ i ] )
YPZhangb34b7e12016-06-14 14:28:19 -070090
Devin Lim142b5342017-07-20 15:22:39 -070091 stepResult = main.testSetUp.envSetup()
Devin Lim58046fa2017-07-05 16:55:00 -070092 # Create DataBase file
93 main.log.info( "Create Database file " + main.dbFileName )
94 resultsDB = open( main.dbFileName, "w+" )
95 resultsDB.close()
96 except Exception as e:
97 main.testSetUp.envSetupException( e )
98 main.testSetUp.evnSetupConclusion( stepResult )
99 main.commit = main.commit.split( " " )[ 1 ]
Jon Hall4ba53f02015-07-29 13:07:41 -0700100 def CASE1( self, main ):
YPZhangb34b7e12016-06-14 14:28:19 -0700101 # Clean up test environment and set up
Jon Hall4ba53f02015-07-29 13:07:41 -0700102 import time
Devin Lim58046fa2017-07-05 16:55:00 -0700103
YPZhangb34b7e12016-06-14 14:28:19 -0700104 main.maxNumBatch = 0
Devin Lim142b5342017-07-20 15:22:39 -0700105 main.testSetUp.ONOSSetUp( main.MN1Ip, main.Cluster, True,
106 cellName=main.cellName, killRemoveMax=False )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700107
YPZhangb34b7e12016-06-14 14:28:19 -0700108 # configure apps
Devin Lim142b5342017-07-20 15:22:39 -0700109 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
110 "deviceCount", value=7 )
111 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
112 "topoShape", value="linear" )
113 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
114 "enabled", value="true" )
115 main.Cluster.active( 0 ).CLI.setCfg( main.intentManagerCfg,
116 "skipReleaseResourcesOnWithdrawal",
117 value="true" )
YPZhange6ef82a2016-07-05 16:48:15 -0700118 if main.flowObj:
Devin Lim142b5342017-07-20 15:22:39 -0700119 main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
120 "useFlowObjectives", value="true" )
121 main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
122 "defaultFlowObjectiveCompiler",
123 value=main.linkCollectionIntentCfg )
Devin Lim58046fa2017-07-05 16:55:00 -0700124 time.sleep( main.startUpSleep )
Jon Hall4ba53f02015-07-29 13:07:41 -0700125
YPZhangb34b7e12016-06-14 14:28:19 -0700126 # balanceMasters
Devin Lim142b5342017-07-20 15:22:39 -0700127 main.Cluster.active( 0 ).CLI.balanceMasters()
Devin Lim58046fa2017-07-05 16:55:00 -0700128 time.sleep( main.startUpSleep )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700129
130 def CASE2( self, main ):
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700131 import time
132 import numpy
YPZhangb34b7e12016-06-14 14:28:19 -0700133 import json
Devin Lim58046fa2017-07-05 16:55:00 -0700134 print( main.intentsList )
YPZhangb34b7e12016-06-14 14:28:19 -0700135 for batchSize in main.intentsList:
Devin Lim58046fa2017-07-05 16:55:00 -0700136 main.log.report( "Intent Batch size: {}".format( batchSize ) )
YPZhangb34b7e12016-06-14 14:28:19 -0700137 main.installLatList = []
138 main.withdrawLatList = []
139 validrun = 0
140 invalidrun = 0
141 # we use two variables to control the iteration
142 while validrun <= main.warmUp + main.sampleSize and invalidrun < 20:
143 if validrun >= main.warmUp:
Devin Lim58046fa2017-07-05 16:55:00 -0700144 main.log.info( "================================================" )
145 main.log.info( "Starting test iteration " + str( validrun - main.warmUp ) )
146 main.log.info( "Total test iteration: " + str( invalidrun + validrun ) )
147 main.log.info( "================================================" )
YPZhangb34b7e12016-06-14 14:28:19 -0700148 else:
Devin Lim58046fa2017-07-05 16:55:00 -0700149 main.log.info( "====================Warm Up=====================" )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700150
YPZhangb34b7e12016-06-14 14:28:19 -0700151 # push intents
Devin Lim142b5342017-07-20 15:22:39 -0700152 installResult = main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
153 main.egress,
154 batchSize,
155 offset=1,
156 options="-i",
157 timeout=main.timeout,
158 getResponse=True )
Devin Lim58046fa2017-07-05 16:55:00 -0700159 if type( installResult ) is str:
YPZhangb34b7e12016-06-14 14:28:19 -0700160 if "Failure" in installResult:
Devin Lim58046fa2017-07-05 16:55:00 -0700161 main.log.error( "Install Intents failure, ignore this iteration." )
YPZhangb34b7e12016-06-14 14:28:19 -0700162 if validrun < main.warmUp:
163 validrun += 1
164 continue
165 else:
166 invalidrun += 1
167 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700168
YPZhangb34b7e12016-06-14 14:28:19 -0700169 try:
Devin Lim58046fa2017-07-05 16:55:00 -0700170 latency = int( installResult.split()[ 5 ] )
171 main.log.info( installResult )
YPZhangb34b7e12016-06-14 14:28:19 -0700172 except:
Devin Lim58046fa2017-07-05 16:55:00 -0700173 main.log.error( "Failed to get latency, ignore this iteration." )
174 main.log.error( "Response from ONOS:" )
175 print( installResult )
YPZhangb34b7e12016-06-14 14:28:19 -0700176 if validrun < main.warmUp:
177 validrun += 1
178 continue
179 else:
180 invalidrun += 1
181 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700182
YPZhangb34b7e12016-06-14 14:28:19 -0700183 if validrun >= main.warmUp:
Devin Lim58046fa2017-07-05 16:55:00 -0700184 main.installLatList.append( latency )
YPZhangb34b7e12016-06-14 14:28:19 -0700185 else:
186 invalidrun += 1
187 continue
Devin Lim58046fa2017-07-05 16:55:00 -0700188 time.sleep( 2 )
YPZhangb34b7e12016-06-14 14:28:19 -0700189 # Withdraw Intents
Devin Lim142b5342017-07-20 15:22:39 -0700190 withdrawResult = main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
191 main.egress,
192 batchSize,
193 offset=1,
194 options="-w",
195 timeout=main.timeout,
196 getResponse=True )
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700197
Devin Lim58046fa2017-07-05 16:55:00 -0700198 if type( withdrawResult ) is str:
YPZhangb34b7e12016-06-14 14:28:19 -0700199 if "Failure" in withdrawResult:
Devin Lim58046fa2017-07-05 16:55:00 -0700200 main.log.error( "withdraw Intents failure, ignore this iteration." )
YPZhangb34b7e12016-06-14 14:28:19 -0700201 if validrun < main.warmUp:
202 validrun += 1
203 continue
204 else:
205 invalidrun += 1
206 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700207
YPZhangb34b7e12016-06-14 14:28:19 -0700208 try:
Devin Lim58046fa2017-07-05 16:55:00 -0700209 latency = int( withdrawResult.split()[ 5 ] )
210 main.log.info( withdrawResult )
YPZhangb34b7e12016-06-14 14:28:19 -0700211 except:
Devin Lim58046fa2017-07-05 16:55:00 -0700212 main.log.error( "Failed to get latency, ignore this iteration." )
213 main.log.error( "Response from ONOS:" )
214 print( withdrawResult )
YPZhangb34b7e12016-06-14 14:28:19 -0700215 if validrun < main.warmUp:
216 validrun += 1
217 continue
218 else:
219 invalidrun += 1
220 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700221
YPZhangb34b7e12016-06-14 14:28:19 -0700222 if validrun >= main.warmUp:
Devin Lim58046fa2017-07-05 16:55:00 -0700223 main.withdrawLatList.append( latency )
YPZhangb34b7e12016-06-14 14:28:19 -0700224 else:
225 invalidrun += 1
226 continue
Devin Lim58046fa2017-07-05 16:55:00 -0700227 time.sleep( 2 )
Devin Lim142b5342017-07-20 15:22:39 -0700228 main.Cluster.active( 0 ).CLI.purgeWithdrawnIntents()
YPZhangb34b7e12016-06-14 14:28:19 -0700229 validrun += 1
Devin Lim58046fa2017-07-05 16:55:00 -0700230 installave = numpy.average( main.installLatList )
231 installstd = numpy.std( main.installLatList )
232 withdrawave = numpy.average( main.withdrawLatList )
233 withdrawstd = numpy.std( main.withdrawLatList )
YPZhangb34b7e12016-06-14 14:28:19 -0700234 # log report
Devin Lim58046fa2017-07-05 16:55:00 -0700235 main.log.report( "----------------------------------------------------" )
Devin Lim142b5342017-07-20 15:22:39 -0700236 main.log.report( "Scale: " + str( main.Cluster.numCtrls ) )
Devin Lim58046fa2017-07-05 16:55:00 -0700237 main.log.report( "Intent batch: " + str( batchSize ) )
238 main.log.report( "Install average: {} std: {}".format( installave, installstd ) )
239 main.log.report( "Withdraw average: {} std: {}".format( withdrawave, withdrawstd ) )
YPZhangb34b7e12016-06-14 14:28:19 -0700240 # write result to database file
Devin Lim58046fa2017-07-05 16:55:00 -0700241 if not ( numpy.isnan( installave ) or numpy.isnan( installstd ) or\
242 numpy.isnan( withdrawstd ) or numpy.isnan( withdrawave ) ):
YPZhangb34b7e12016-06-14 14:28:19 -0700243 databaseString = "'" + main.commit + "',"
Devin Lim142b5342017-07-20 15:22:39 -0700244 databaseString += str( main.Cluster.numCtrls ) + ","
Devin Lim58046fa2017-07-05 16:55:00 -0700245 databaseString += str( batchSize ) + ","
246 databaseString += str( installave ) + ","
247 databaseString += str( installstd ) + ","
248 databaseString += str( withdrawave ) + ","
249 databaseString += str( withdrawstd ) + "\n"
250 resultsDB = open( main.dbFileName, "a" )
251 resultsDB.write( databaseString )
YPZhangb34b7e12016-06-14 14:28:19 -0700252 resultsDB.close()