blob: f5f1f198e63e85e4d616edd41039ab2b5abc0950 [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2015 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
YPZhangfebf7302016-05-24 16:45:56 -070021# SCPFintentRerouteLat
22"""
23SCPFintentRerouteLat
24 - Test Intent Reroute Latency
25 - Test Algorithm:
26 1. Start Null Provider reroute Topology
27 2. Using Push-test-intents to push batch size intents from switch 1 to switch 7
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070028 3. Cut the link between switch 3 and switch 4 ( the path will reroute to switch 8 )
YPZhangfebf7302016-05-24 16:45:56 -070029 4. Get the topology time stamp
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070030 5. Get Intent reroute( Installed ) time stamp from each nodes
YPZhangfebf7302016-05-24 16:45:56 -070031 6. Use the latest intent time stamp subtract topology time stamp
32 - This test will run 5 warm up by default, warm up iteration can be setup in Param file
33 - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file
34 - The unit of the latency result is milliseconds
35"""
cameron@onlab.us78b89652015-07-08 15:21:03 -070036class SCPFintentRerouteLat:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070037
Devin Lim142b5342017-07-20 15:22:39 -070038 def __init__( self ):
cameron@onlab.us78b89652015-07-08 15:21:03 -070039 self.default = ''
40
YPZhangfebf7302016-05-24 16:45:56 -070041 def CASE0( self, main ):
Chiyu Chengec63bde2016-11-17 18:11:36 -080042 import imp
43 import os
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070044 """
YPZhangfebf7302016-05-24 16:45:56 -070045 - GIT
46 - BUILDING ONOS
47 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
48 This step is usually skipped. Because in a Jenkins driven automated
49 test env. We want Jenkins jobs to pull&build for flexibility to handle
50 different versions of ONOS.
51 - Construct tests variables
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070052 """
Devin Lim58046fa2017-07-05 16:55:00 -070053 try:
54 from tests.dependencies.ONOSSetup import ONOSSetup
55 main.testSetUp = ONOSSetup()
56 except ImportError:
57 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070058 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070059 main.testSetUp.envSetupDescription()
60 stepResult = main.FALSE
61 try:
Devin Lim58046fa2017-07-05 16:55:00 -070062 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
63 main.BENCHUser = main.params[ 'BENCH' ][ 'user' ]
64 main.BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
65 main.MN1Ip = main.params[ 'MN' ][ 'ip1' ]
Devin Lim58046fa2017-07-05 16:55:00 -070066 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
Devin Lim142b5342017-07-20 15:22:39 -070067 main.scale = ( main.params[ 'SCALE' ] ).split( "," )
Devin Lim58046fa2017-07-05 16:55:00 -070068 main.timeout = int( main.params[ 'SLEEP' ][ 'timeout' ] )
69 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
70 main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] )
71 main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] )
72 main.setMasterSleep = int( main.params[ 'SLEEP' ][ 'setmaster' ] )
73 main.verifyAttempts = int( main.params[ 'ATTEMPTS' ][ 'verify' ] )
74 main.maxInvalidRun = int( main.params[ 'ATTEMPTS' ][ 'maxInvalidRun' ] )
Devin Lime6fe3c42017-10-18 16:28:40 -070075 main.cfgRetry = int( main.params[ 'ATTEMPTS' ][ 'cfg' ] )
Devin Lim58046fa2017-07-05 16:55:00 -070076 main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
Devin Lim142b5342017-07-20 15:22:39 -070077 main.intentManagerCfg = main.params[ 'CFG' ][ 'intentManager' ]
78 main.intentConfigRegiCfg = main.params[ 'CFG' ][ 'intentConfigRegi' ]
79 main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ]
80 main.linkCollectionIntentCfg = main.params[ 'CFG' ][ 'linkCollectionIntent' ]
Devin Lim58046fa2017-07-05 16:55:00 -070081 main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
82 main.ingress = main.params[ 'TEST' ][ 'ingress' ]
83 main.egress = main.params[ 'TEST' ][ 'egress' ]
84 main.debug = main.params[ 'TEST' ][ 'debug' ]
85 main.flowObj = main.params[ 'TEST' ][ 'flowObj' ]
86 main.deviceCount = int( main.params[ 'TEST' ][ 'deviceCount' ] )
87 main.end1 = main.params[ 'TEST' ][ 'end1' ]
88 main.end2 = main.params[ 'TEST' ][ 'end2' ]
89 main.searchTerm = main.params[ 'SEARCHTERM' ]
90 if main.flowObj == "True":
91 main.flowObj = True
92 main.dbFileName = main.params[ 'DATABASE' ][ 'dbFlowObj' ]
93 main.intentsList = ( main.params[ 'TEST' ][ 'FObjintents' ] ).split( "," )
94 else:
95 main.flowObj = False
96 main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ]
97 main.intentsList = ( main.params[ 'TEST' ][ 'intents' ] ).split( "," )
cameron@onlab.us78b89652015-07-08 15:21:03 -070098
Devin Lim142b5342017-07-20 15:22:39 -070099 stepResult = main.testSetUp.envSetup()
YPZhangfebf7302016-05-24 16:45:56 -0700100
Devin Lim142b5342017-07-20 15:22:39 -0700101 for i in range( 0, len( main.intentsList ) ):
Devin Lim58046fa2017-07-05 16:55:00 -0700102 main.intentsList[ i ] = int( main.intentsList[ i ] )
103 # Create DataBase file
104 main.log.info( "Create Database file " + main.dbFileName )
105 resultsDB = open( main.dbFileName, "w+" )
106 resultsDB.close()
107 file1 = main.params[ "DEPENDENCY" ][ "FILE1" ]
108 main.dependencyPath = os.path.dirname( os.getcwd() ) + main.params[ "DEPENDENCY" ][ "PATH" ]
109 main.intentRerouteLatFuncs = imp.load_source( file1, main.dependencyPath + file1 + ".py" )
YPZhangfebf7302016-05-24 16:45:56 -0700110
Devin Lim58046fa2017-07-05 16:55:00 -0700111 main.record = 0
112 except Exception as e:
113 main.testSetUp.envSetupException( e )
114 main.testSetUp.evnSetupConclusion( stepResult )
You Wang47b91832017-08-02 11:36:13 -0700115 main.commit = main.commit.split( " " )[ 1 ]
YPZhangfebf7302016-05-24 16:45:56 -0700116
117 def CASE1( self, main ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700118 """
YPZhangfebf7302016-05-24 16:45:56 -0700119 clean up test environment and set up
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700120 """
cameron@onlab.us78b89652015-07-08 15:21:03 -0700121 import time
122
YPZhangfebf7302016-05-24 16:45:56 -0700123 main.maxNumBatch = 0
You Wanga0f6ff62018-01-11 15:46:30 -0800124 main.testSetUp.ONOSSetUp( main.Cluster, True,
Devin Lim142b5342017-07-20 15:22:39 -0700125 cellName=main.cellName, killRemoveMax=False )
Devin Lime6fe3c42017-10-18 16:28:40 -0700126 configRetry = 0
127 main.cfgCheck = False
128 while configRetry < main.cfgRetry:
129 # configure apps
130 stepResult = main.TRUE
131 stepResult = stepResult and \
132 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
133 "deviceCount",
134 value=main.deviceCount )
135
136 stepResult = stepResult and \
137 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
138 "topoShape",
139 value="reroute" )
140 stepResult = stepResult and \
141 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg,
142 "enabled",
143 value="true" )
144
145 stepResult = stepResult and \
146 main.Cluster.active( 0 ).CLI.setCfg( main.intentManagerCfg,
147 "skipReleaseResourcesOnWithdrawal",
148 value="true" )
149 if main.flowObj:
150 stepResult = stepResult and \
151 main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
152 "useFlowObjectives",
153 value="true" )
154 stepResult = stepResult and \
155 main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
156 "defaultFlowObjectiveCompiler",
157 value=main.linkCollectionIntentCfg )
158 if stepResult:
159 main.cfgCheck = True
160 break
161 configRetry += 1
162 time.sleep( main.verifySleep )
163
Chiyu Chengec63bde2016-11-17 18:11:36 -0800164 time.sleep( main.startUpSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700165 for ctrl in main.Cluster.active():
166 ctrl.CLI.logSet( "DEBUG", "org.onosproject.metrics.topology" )
167 ctrl.CLI.logSet( "DEBUG", "org.onosproject.metrics.intent" )
YPZhangfebf7302016-05-24 16:45:56 -0700168 # Balance Master
Devin Lim142b5342017-07-20 15:22:39 -0700169 main.Cluster.active( 0 ).CLI.balanceMasters()
You Wang0b9039d2017-01-12 16:51:29 -0800170 time.sleep( main.setMasterSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700171 if main.Cluster.numCtrls:
172 main.Cluster.active( 0 ).CLI.deviceRole( main.end1[ 'name' ], main.Cluster.active( 0 ).ipAddress )
173 main.Cluster.active( 0 ).CLI.deviceRole( main.end2[ 'name' ], main.Cluster.active( 0 ).ipAddress )
YPZhang2b9b26d2016-06-20 16:18:29 -0700174 time.sleep( main.setMasterSleep )
Devin Lime6fe3c42017-10-18 16:28:40 -0700175 if not main.cfgCheck:
176 main.log.error( "Setting configuration to the ONOS failed. Skip the rest of the steps" )
cameron@onlab.us78b89652015-07-08 15:21:03 -0700177
178 def CASE2( self, main ):
cameron@onlab.us78b89652015-07-08 15:21:03 -0700179 import time
180 import numpy
181 import datetime
YPZhangfebf7302016-05-24 16:45:56 -0700182 import json
183 # from scipy import stats
Devin Lime6fe3c42017-10-18 16:28:40 -0700184 testResult = main.TRUE
185 main.case( "Intent Reroute starts" )
186 main.step( "Checking intent reroute" )
187 if main.cfgCheck:
188 print( main.intentsList )
189 for batchSize in main.intentsList:
190 main.batchSize = batchSize
191 main.log.report( "Intent Batch size: " + str( batchSize ) + "\n " )
192 firstLocalLatencies = []
193 lastLocalLatencies = []
194 firstGlobalLatencies = []
195 lastGlobalLatencies = []
196 main.startLine = {}
197 main.validRun = 0
198 main.invalidRun = 0
199 while main.validRun <= main.warmUp + main.sampleSize and main.invalidRun <= main.maxInvalidRun:
200 if main.validRun >= main.warmUp:
201 main.log.info( "================================================" )
202 main.log.info( "Valid iteration: {} ".format( main.validRun - main.warmUp ) )
203 main.log.info( "Total iteration: {}".format( main.validRun + main.invalidRun ) )
204 main.log.info( "================================================" )
205 else:
206 main.log.info( "====================Warm Up=====================" )
cameron@onlab.us78b89652015-07-08 15:21:03 -0700207
Devin Lime6fe3c42017-10-18 16:28:40 -0700208 # push intents
209 main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
210 main.egress,
211 main.batchSize,
212 offset=1,
213 options="-i",
214 timeout=main.timeout )
cameron@onlab.us78b89652015-07-08 15:21:03 -0700215
Devin Lime6fe3c42017-10-18 16:28:40 -0700216 # check links, flows and intents
217 main.intentRerouteLatFuncs.sanityCheck( main,
218 main.deviceCount * 2,
219 batchSize * ( main.deviceCount - 1 ),
220 main.batchSize )
221 if not main.verify:
222 main.log.warn( "Sanity check failed, skipping this iteration..." )
223 continue
cameron@onlab.us78b89652015-07-08 15:21:03 -0700224
Devin Lime6fe3c42017-10-18 16:28:40 -0700225 # Insert one line in karaf.log before link down
226 main.Cluster.command( "log",
227 args=[ "\'Scale: {}, Batch:{}, Iteration: {}\'".format(
228 main.Cluster.numCtrls, batchSize, main.validRun + main.invalidRun ) ],
229 returnBool=True, specificDriver=2 )
230 # bring link down
231 main.Cluster.active( 0 ).CLI.link( main.end1[ 'port' ], main.end2[ 'port' ], "down",
232 timeout=main.timeout, showResponse=False )
You Wang6d301d42017-04-21 10:49:33 -0700233
Devin Lime6fe3c42017-10-18 16:28:40 -0700234 # check links, flows and intents
235 main.intentRerouteLatFuncs.sanityCheck( main,
236 ( main.deviceCount - 1 ) * 2,
237 batchSize * main.deviceCount,
238 main.batchSize )
239 if not main.verify:
240 main.log.warn( "Sanity check failed, skipping this iteration..." )
241 continue
You Wang6d301d42017-04-21 10:49:33 -0700242
Devin Lime6fe3c42017-10-18 16:28:40 -0700243 # Get timestamp of last LINK_REMOVED event as separator between iterations
244 skip = False
245 for i in range( main.Cluster.numCtrls ):
You Wang6d301d42017-04-21 10:49:33 -0700246 logNum = main.intentRerouteLatFuncs.getLogNum( main, i )
Devin Lim142b5342017-07-20 15:22:39 -0700247 timestamp = str( main.Cluster.active( i ).CLI.getTimeStampFromLog( "last",
248 "LINK_REMOVED",
249 "time = ", " ",
250 logNum=logNum ) )
Devin Lime6fe3c42017-10-18 16:28:40 -0700251 if timestamp == main.ERROR:
252 # Try again in case that the log number just increased
253 logNum = main.intentRerouteLatFuncs.getLogNum( main, i )
254 timestamp = str( main.Cluster.active( i ).CLI.getTimeStampFromLog( "last",
255 "LINK_REMOVED",
256 "time = ", " ",
257 logNum=logNum ) )
258 if timestamp == main.ERROR:
259 main.log.warn( "Cannot find the event we want in the log, skipping this iteration..." )
260 main.intentRerouteLatFuncs.bringBackTopology( main )
261 if main.validRun >= main.warmUp:
262 main.invalidRun += 1
263 else:
264 main.validRun += 1
265 skip = True
266 break
267 else:
268 main.startLine[ i ] = timestamp
269 main.log.info( "Timestamp of last LINK_REMOVED event on node {} is {}".format( i + 1,
270 main.startLine[ i ] ) )
271 if skip:
272 continue
273
274 # calculate values
275 topologyTimestamps = main.intentRerouteLatFuncs.getTopologyTimestamps( main )
276 intentTimestamps = main.intentRerouteLatFuncs.getIntentTimestamps( main )
277 if intentTimestamps == main.ERROR or topologyTimestamps == main.ERROR:
278 main.log.info( "Got invalid timestamp, skipping this iteration..." )
You Wang6d301d42017-04-21 10:49:33 -0700279 main.intentRerouteLatFuncs.bringBackTopology( main )
280 if main.validRun >= main.warmUp:
281 main.invalidRun += 1
282 else:
283 main.validRun += 1
Devin Lime6fe3c42017-10-18 16:28:40 -0700284 continue
You Wang6d301d42017-04-21 10:49:33 -0700285 else:
Devin Lime6fe3c42017-10-18 16:28:40 -0700286 main.log.info( "Got valid timestamps" )
You Wang6d301d42017-04-21 10:49:33 -0700287
Devin Lime6fe3c42017-10-18 16:28:40 -0700288 firstLocalLatnecy, lastLocalLatnecy, firstGlobalLatency, lastGlobalLatnecy \
289 = main.intentRerouteLatFuncs.calculateLatency( main, topologyTimestamps, intentTimestamps )
290 if firstLocalLatnecy < 0:
291 main.log.info( "Got negative latency, skipping this iteration..." )
292 main.intentRerouteLatFuncs.bringBackTopology( main )
293 if main.validRun >= main.warmUp:
294 main.invalidRun += 1
295 else:
296 main.validRun += 1
297 continue
YPZhang8742d2e2016-06-16 15:31:58 -0700298 else:
Devin Lime6fe3c42017-10-18 16:28:40 -0700299 main.log.info( "Got valid latencies" )
Chiyu Chengec63bde2016-11-17 18:11:36 -0800300 main.validRun += 1
You Wang6d301d42017-04-21 10:49:33 -0700301
You Wang6d301d42017-04-21 10:49:33 -0700302 if main.validRun >= main.warmUp:
Devin Lime6fe3c42017-10-18 16:28:40 -0700303 firstLocalLatencies.append( firstLocalLatnecy )
304 lastLocalLatencies.append( lastLocalLatnecy )
305 firstGlobalLatencies.append( firstGlobalLatency )
306 lastGlobalLatencies.append( lastGlobalLatnecy )
YPZhangfebf7302016-05-24 16:45:56 -0700307
Devin Lime6fe3c42017-10-18 16:28:40 -0700308 # bring up link and withdraw intents
309 main.Cluster.active( 0 ).CLI.link( main.end1[ 'port' ],
310 main.end2[ 'port' ],
311 "up",
312 timeout=main.timeout )
313 main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
314 main.egress,
315 batchSize,
316 offset=1,
317 options="-w",
318 timeout=main.timeout )
319 main.Cluster.active( 0 ).CLI.purgeWithdrawnIntents()
You Wang6d301d42017-04-21 10:49:33 -0700320
Devin Lime6fe3c42017-10-18 16:28:40 -0700321 # check links, flows and intents
322 main.intentRerouteLatFuncs.sanityCheck( main, main.deviceCount * 2, 0, 0 )
323 if not main.verify:
324 main.log.warn( "Sanity check failed, skipping this iteration..." )
325 continue
326 result = ( main.TRUE if main.invalidRun <= main.maxInvalidRun else main.FALSE )
327 aveLocalLatency = numpy.average( lastLocalLatencies ) if lastLocalLatencies and result else 0
328 aveGlobalLatency = numpy.average( lastGlobalLatencies ) if lastGlobalLatencies and result else 0
329 stdLocalLatency = numpy.std( lastLocalLatencies ) if lastLocalLatencies and result else 0
330 stdGlobalLatency = numpy.std( lastGlobalLatencies ) if lastGlobalLatencies and result else 0
331 testResult = testResult and result
You Wang6d301d42017-04-21 10:49:33 -0700332
Devin Lime6fe3c42017-10-18 16:28:40 -0700333 main.log.report( "Scale: " + str( main.Cluster.numCtrls ) + " \tIntent batch: " + str( batchSize ) )
334 main.log.report( "Local latency average:................" + str( aveLocalLatency ) )
335 main.log.report( "Global latency average:................" + str( aveGlobalLatency ) )
336 main.log.report( "Local latency std:................" + str( stdLocalLatency ) )
337 main.log.report( "Global latency std:................" + str( stdGlobalLatency ) )
338 main.log.report( "________________________________________________________" )
YPZhangfebf7302016-05-24 16:45:56 -0700339
Devin Lime6fe3c42017-10-18 16:28:40 -0700340 if not ( numpy.isnan( aveLocalLatency ) or numpy.isnan( aveGlobalLatency ) ):
341 # check if got NaN for result
342 resultsDB = open( main.dbFileName, "a" )
343 resultsDB.write( "'" + main.commit + "'," )
344 resultsDB.write( str( main.Cluster.numCtrls ) + "," )
345 resultsDB.write( str( batchSize ) + "," )
346 resultsDB.write( str( aveLocalLatency ) + "," )
347 resultsDB.write( str( stdLocalLatency ) + "\n" )
348 resultsDB.close()
349 else:
350 testResult = main.FALSE
351 utilities.assert_equals( expect=main.TRUE,
352 actual=testResult,
353 onpass="Installing and withdrawing intents reroute properly",
354 onfail="There was something wrong installing and withdrawing intents reroute" )