blob: 285e12d637250451946a118fa94e34d50512104e [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
2Copyright 2015 Open Networking Foundation (ONF)
3
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
11 (at your option) any later version.
12
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
21
YPZhangfebf7302016-05-24 16:45:56 -070022# SCPFintentRerouteLat
23"""
24SCPFintentRerouteLat
25 - Test Intent Reroute Latency
26 - Test Algorithm:
27 1. Start Null Provider reroute Topology
28 2. Using Push-test-intents to push batch size intents from switch 1 to switch 7
29 3. Cut the link between switch 3 and switch 4 (the path will reroute to switch 8)
30 4. Get the topology time stamp
31 5. Get Intent reroute(Installed) time stamp from each nodes
32 6. Use the latest intent time stamp subtract topology time stamp
33 - This test will run 5 warm up by default, warm up iteration can be setup in Param file
34 - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file
35 - The unit of the latency result is milliseconds
36"""
cameron@onlab.us78b89652015-07-08 15:21:03 -070037
Chiyu Chengec63bde2016-11-17 18:11:36 -080038
cameron@onlab.us78b89652015-07-08 15:21:03 -070039class SCPFintentRerouteLat:
Devin Lim142b5342017-07-20 15:22:39 -070040 def __init__( self ):
cameron@onlab.us78b89652015-07-08 15:21:03 -070041 self.default = ''
42
YPZhangfebf7302016-05-24 16:45:56 -070043 def CASE0( self, main ):
Chiyu Chengec63bde2016-11-17 18:11:36 -080044 import imp
45 import os
YPZhangfebf7302016-05-24 16:45:56 -070046 '''
47 - GIT
48 - BUILDING ONOS
49 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
50 This step is usually skipped. Because in a Jenkins driven automated
51 test env. We want Jenkins jobs to pull&build for flexibility to handle
52 different versions of ONOS.
53 - Construct tests variables
54 '''
Devin Lim58046fa2017-07-05 16:55:00 -070055 try:
56 from tests.dependencies.ONOSSetup import ONOSSetup
57 main.testSetUp = ONOSSetup()
58 except ImportError:
59 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070060 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070061 main.testSetUp.envSetupDescription()
62 stepResult = main.FALSE
63 try:
Devin Lim58046fa2017-07-05 16:55:00 -070064 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
65 main.BENCHUser = main.params[ 'BENCH' ][ 'user' ]
66 main.BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
67 main.MN1Ip = main.params[ 'MN' ][ 'ip1' ]
Devin Lim58046fa2017-07-05 16:55:00 -070068 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
Devin Lim142b5342017-07-20 15:22:39 -070069 main.scale = ( main.params[ 'SCALE' ] ).split( "," )
Devin Lim58046fa2017-07-05 16:55:00 -070070 main.timeout = int( main.params[ 'SLEEP' ][ 'timeout' ] )
71 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
72 main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] )
73 main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] )
74 main.setMasterSleep = int( main.params[ 'SLEEP' ][ 'setmaster' ] )
75 main.verifyAttempts = int( main.params[ 'ATTEMPTS' ][ 'verify' ] )
76 main.maxInvalidRun = int( main.params[ 'ATTEMPTS' ][ 'maxInvalidRun' ] )
77 main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
Devin Lim142b5342017-07-20 15:22:39 -070078 main.intentManagerCfg = main.params[ 'CFG' ][ 'intentManager' ]
79 main.intentConfigRegiCfg = main.params[ 'CFG' ][ 'intentConfigRegi' ]
80 main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ]
81 main.linkCollectionIntentCfg = main.params[ 'CFG' ][ 'linkCollectionIntent' ]
Devin Lim58046fa2017-07-05 16:55:00 -070082 main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
83 main.ingress = main.params[ 'TEST' ][ 'ingress' ]
84 main.egress = main.params[ 'TEST' ][ 'egress' ]
85 main.debug = main.params[ 'TEST' ][ 'debug' ]
86 main.flowObj = main.params[ 'TEST' ][ 'flowObj' ]
87 main.deviceCount = int( main.params[ 'TEST' ][ 'deviceCount' ] )
88 main.end1 = main.params[ 'TEST' ][ 'end1' ]
89 main.end2 = main.params[ 'TEST' ][ 'end2' ]
90 main.searchTerm = main.params[ 'SEARCHTERM' ]
91 if main.flowObj == "True":
92 main.flowObj = True
93 main.dbFileName = main.params[ 'DATABASE' ][ 'dbFlowObj' ]
94 main.intentsList = ( main.params[ 'TEST' ][ 'FObjintents' ] ).split( "," )
95 else:
96 main.flowObj = False
97 main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ]
98 main.intentsList = ( main.params[ 'TEST' ][ 'intents' ] ).split( "," )
cameron@onlab.us78b89652015-07-08 15:21:03 -070099
Devin Lim142b5342017-07-20 15:22:39 -0700100 stepResult = main.testSetUp.envSetup()
YPZhangfebf7302016-05-24 16:45:56 -0700101
Devin Lim142b5342017-07-20 15:22:39 -0700102 for i in range( 0, len( main.intentsList ) ):
Devin Lim58046fa2017-07-05 16:55:00 -0700103 main.intentsList[ i ] = int( main.intentsList[ i ] )
104 # Create DataBase file
105 main.log.info( "Create Database file " + main.dbFileName )
106 resultsDB = open( main.dbFileName, "w+" )
107 resultsDB.close()
108 file1 = main.params[ "DEPENDENCY" ][ "FILE1" ]
109 main.dependencyPath = os.path.dirname( os.getcwd() ) + main.params[ "DEPENDENCY" ][ "PATH" ]
110 main.intentRerouteLatFuncs = imp.load_source( file1, main.dependencyPath + file1 + ".py" )
YPZhangfebf7302016-05-24 16:45:56 -0700111
Devin Lim58046fa2017-07-05 16:55:00 -0700112 main.record = 0
113 except Exception as e:
114 main.testSetUp.envSetupException( e )
115 main.testSetUp.evnSetupConclusion( stepResult )
You Wang47b91832017-08-02 11:36:13 -0700116 main.commit = main.commit.split( " " )[ 1 ]
YPZhangfebf7302016-05-24 16:45:56 -0700117
Devin Lim142b5342017-07-20 15:22:39 -0700118
YPZhangfebf7302016-05-24 16:45:56 -0700119 def CASE1( self, main ):
120 '''
121 clean up test environment and set up
122 '''
cameron@onlab.us78b89652015-07-08 15:21:03 -0700123 import time
124
YPZhangfebf7302016-05-24 16:45:56 -0700125 main.maxNumBatch = 0
Devin Lim142b5342017-07-20 15:22:39 -0700126 main.testSetUp.ONOSSetUp( main.MN1Ip, main.Cluster, True,
127 cellName=main.cellName, killRemoveMax=False )
YPZhangfebf7302016-05-24 16:45:56 -0700128 # configure apps
Devin Lim142b5342017-07-20 15:22:39 -0700129 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg, "deviceCount", value=main.deviceCount )
130 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg, "topoShape", value="reroute" )
131 main.Cluster.active( 0 ).CLI.setCfg( main.nullProviderCfg, "enabled", value="true" )
132 main.Cluster.active( 0 ).CLI.setCfg( main.intentManagerCfg, "skipReleaseResourcesOnWithdrawal",
133 value="true" )
YPZhange6ef82a2016-07-05 16:48:15 -0700134 if main.flowObj:
Devin Lim142b5342017-07-20 15:22:39 -0700135 main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
136 "useFlowObjectives", value="true" )
137 main.Cluster.active( 0 ).CLI.setCfg( main.intentConfigRegiCfg,
138 "defaultFlowObjectiveCompiler",
139 value=main.linkCollectionIntentCfg )
Chiyu Chengec63bde2016-11-17 18:11:36 -0800140 time.sleep( main.startUpSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700141 for ctrl in main.Cluster.active():
142 ctrl.CLI.logSet( "DEBUG", "org.onosproject.metrics.topology" )
143 ctrl.CLI.logSet( "DEBUG", "org.onosproject.metrics.intent" )
YPZhangfebf7302016-05-24 16:45:56 -0700144 # Balance Master
Devin Lim142b5342017-07-20 15:22:39 -0700145 main.Cluster.active( 0 ).CLI.balanceMasters()
You Wang0b9039d2017-01-12 16:51:29 -0800146 time.sleep( main.setMasterSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700147 if main.Cluster.numCtrls:
148 main.Cluster.active( 0 ).CLI.deviceRole( main.end1[ 'name' ], main.Cluster.active( 0 ).ipAddress )
149 main.Cluster.active( 0 ).CLI.deviceRole( main.end2[ 'name' ], main.Cluster.active( 0 ).ipAddress )
YPZhang2b9b26d2016-06-20 16:18:29 -0700150 time.sleep( main.setMasterSleep )
cameron@onlab.us78b89652015-07-08 15:21:03 -0700151
152 def CASE2( self, main ):
cameron@onlab.us78b89652015-07-08 15:21:03 -0700153 import time
154 import numpy
155 import datetime
YPZhangfebf7302016-05-24 16:45:56 -0700156 import json
157 # from scipy import stats
cameron@onlab.us78b89652015-07-08 15:21:03 -0700158
Devin Lim142b5342017-07-20 15:22:39 -0700159 print( main.intentsList )
YPZhangfebf7302016-05-24 16:45:56 -0700160 for batchSize in main.intentsList:
You Wang0b9039d2017-01-12 16:51:29 -0800161 main.batchSize = batchSize
Devin Lim142b5342017-07-20 15:22:39 -0700162 main.log.report( "Intent Batch size: " + str( batchSize ) + "\n " )
You Wang6d301d42017-04-21 10:49:33 -0700163 firstLocalLatencies = []
164 lastLocalLatencies = []
165 firstGlobalLatencies = []
166 lastGlobalLatencies = []
167 main.startLine = {}
Chiyu Chengec63bde2016-11-17 18:11:36 -0800168 main.validRun = 0
169 main.invalidRun = 0
You Wang0b9039d2017-01-12 16:51:29 -0800170 while main.validRun <= main.warmUp + main.sampleSize and main.invalidRun <= main.maxInvalidRun:
Chiyu Chengec63bde2016-11-17 18:11:36 -0800171 if main.validRun >= main.warmUp:
Devin Lim142b5342017-07-20 15:22:39 -0700172 main.log.info( "================================================" )
173 main.log.info( "Valid iteration: {} ".format( main.validRun - main.warmUp ) )
174 main.log.info( "Total iteration: {}".format( main.validRun + main.invalidRun ) )
175 main.log.info( "================================================" )
cameron@onlab.us78b89652015-07-08 15:21:03 -0700176 else:
Devin Lim142b5342017-07-20 15:22:39 -0700177 main.log.info( "====================Warm Up=====================" )
cameron@onlab.us78b89652015-07-08 15:21:03 -0700178
YPZhangfebf7302016-05-24 16:45:56 -0700179 # push intents
Devin Lim142b5342017-07-20 15:22:39 -0700180 main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
181 main.egress,
182 main.batchSize,
183 offset=1,
184 options="-i",
185 timeout=main.timeout )
cameron@onlab.us78b89652015-07-08 15:21:03 -0700186
You Wang6d301d42017-04-21 10:49:33 -0700187 # check links, flows and intents
Devin Lim142b5342017-07-20 15:22:39 -0700188 main.intentRerouteLatFuncs.sanityCheck( main,
189 main.deviceCount * 2,
190 batchSize * ( main.deviceCount - 1 ),
191 main.batchSize )
Chiyu Chengec63bde2016-11-17 18:11:36 -0800192 if not main.verify:
You Wang6d301d42017-04-21 10:49:33 -0700193 main.log.warn( "Sanity check failed, skipping this iteration..." )
194 continue
195
196 # Insert one line in karaf.log before link down
Devin Lim142b5342017-07-20 15:22:39 -0700197 main.Cluster.command( "log",
198 args=[ "\'Scale: {}, Batch:{}, Iteration: {}\'".format(
199 main.Cluster.numCtrls, batchSize, main.validRun + main.invalidRun ) ],
200 returnBool=True, specificDriver=2 )
You Wang6d301d42017-04-21 10:49:33 -0700201 # bring link down
Devin Lim142b5342017-07-20 15:22:39 -0700202 main.Cluster.active( 0 ).CLI.link( main.end1[ 'port' ], main.end2[ 'port' ], "down",
203 timeout=main.timeout, showResponse=False )
You Wang6d301d42017-04-21 10:49:33 -0700204
205 # check links, flows and intents
Devin Lim142b5342017-07-20 15:22:39 -0700206 main.intentRerouteLatFuncs.sanityCheck( main,
207 ( main.deviceCount - 1 ) * 2,
208 batchSize * main.deviceCount,
209 main.batchSize )
Chiyu Chengec63bde2016-11-17 18:11:36 -0800210 if not main.verify:
You Wang6d301d42017-04-21 10:49:33 -0700211 main.log.warn( "Sanity check failed, skipping this iteration..." )
212 continue
213
214 # Get timestamp of last LINK_REMOVED event as separator between iterations
215 skip = False
Devin Lim142b5342017-07-20 15:22:39 -0700216 for i in range( main.Cluster.numCtrls ):
You Wang6d301d42017-04-21 10:49:33 -0700217 logNum = main.intentRerouteLatFuncs.getLogNum( main, i )
Devin Lim142b5342017-07-20 15:22:39 -0700218 timestamp = str( main.Cluster.active( i ).CLI.getTimeStampFromLog( "last",
219 "LINK_REMOVED",
220 "time = ", " ",
221 logNum=logNum ) )
You Wang6d301d42017-04-21 10:49:33 -0700222 if timestamp == main.ERROR:
223 # Try again in case that the log number just increased
224 logNum = main.intentRerouteLatFuncs.getLogNum( main, i )
Devin Lim142b5342017-07-20 15:22:39 -0700225 timestamp = str( main.Cluster.active( i ).CLI.getTimeStampFromLog( "last",
226 "LINK_REMOVED",
227 "time = ", " ",
228 logNum=logNum ) )
You Wang6d301d42017-04-21 10:49:33 -0700229 if timestamp == main.ERROR:
230 main.log.warn( "Cannot find the event we want in the log, skipping this iteration..." )
231 main.intentRerouteLatFuncs.bringBackTopology( main )
232 if main.validRun >= main.warmUp:
233 main.invalidRun += 1
234 else:
235 main.validRun += 1
236 skip = True
237 break
238 else:
239 main.startLine[ i ] = timestamp
Devin Lim142b5342017-07-20 15:22:39 -0700240 main.log.info( "Timestamp of last LINK_REMOVED event on node {} is {}".format( i + 1,
241 main.startLine[ i ] ) )
You Wang6d301d42017-04-21 10:49:33 -0700242 if skip: continue
243
244 # calculate values
245 topologyTimestamps = main.intentRerouteLatFuncs.getTopologyTimestamps( main )
246 intentTimestamps = main.intentRerouteLatFuncs.getIntentTimestamps( main )
247 if intentTimestamps == main.ERROR or topologyTimestamps == main.ERROR:
248 main.log.info( "Got invalid timestamp, skipping this iteration..." )
249 main.intentRerouteLatFuncs.bringBackTopology( main )
Chiyu Chengec63bde2016-11-17 18:11:36 -0800250 if main.validRun >= main.warmUp:
251 main.invalidRun += 1
YPZhang8742d2e2016-06-16 15:31:58 -0700252 else:
Chiyu Chengec63bde2016-11-17 18:11:36 -0800253 main.validRun += 1
YPZhangfebf7302016-05-24 16:45:56 -0700254 continue
Chiyu Chengec63bde2016-11-17 18:11:36 -0800255 else:
You Wang6d301d42017-04-21 10:49:33 -0700256 main.log.info( "Got valid timestamps" )
257
Devin Lim142b5342017-07-20 15:22:39 -0700258 firstLocalLatnecy, lastLocalLatnecy, firstGlobalLatency, lastGlobalLatnecy \
259 = main.intentRerouteLatFuncs.calculateLatency( main, topologyTimestamps, intentTimestamps )
You Wang6d301d42017-04-21 10:49:33 -0700260 if firstLocalLatnecy < 0:
261 main.log.info( "Got negative latency, skipping this iteration..." )
262 main.intentRerouteLatFuncs.bringBackTopology( main )
263 if main.validRun >= main.warmUp:
264 main.invalidRun += 1
265 else:
266 main.validRun += 1
267 continue
268 else:
269 main.log.info( "Got valid latencies" )
Chiyu Chengec63bde2016-11-17 18:11:36 -0800270 main.validRun += 1
YPZhangfebf7302016-05-24 16:45:56 -0700271
You Wang6d301d42017-04-21 10:49:33 -0700272 firstLocalLatencies.append( firstLocalLatnecy )
273 lastLocalLatencies.append( lastLocalLatnecy )
274 firstGlobalLatencies.append( firstGlobalLatency )
275 lastGlobalLatencies.append( lastGlobalLatnecy )
276
277 # bring up link and withdraw intents
Devin Lim142b5342017-07-20 15:22:39 -0700278 main.Cluster.active( 0 ).CLI.link( main.end1[ 'port' ],
279 main.end2[ 'port' ],
280 "up",
281 timeout=main.timeout )
282 main.Cluster.active( 0 ).CLI.pushTestIntents( main.ingress,
283 main.egress,
284 batchSize,
285 offset=1,
286 options="-w",
287 timeout=main.timeout )
288 main.Cluster.active( 0 ).CLI.purgeWithdrawnIntents()
You Wang6d301d42017-04-21 10:49:33 -0700289
290 # check links, flows and intents
291 main.intentRerouteLatFuncs.sanityCheck( main, main.deviceCount * 2, 0, 0 )
Chiyu Chengec63bde2016-11-17 18:11:36 -0800292 if not main.verify:
YPZhangfebf7302016-05-24 16:45:56 -0700293 continue
294
You Wang6d301d42017-04-21 10:49:33 -0700295 aveLocalLatency = numpy.average( lastLocalLatencies )
296 aveGlobalLatency = numpy.average( lastGlobalLatencies )
297 stdLocalLatency = numpy.std( lastLocalLatencies )
298 stdGlobalLatency = numpy.std( lastGlobalLatencies )
Chiyu Chengec63bde2016-11-17 18:11:36 -0800299
Devin Lim142b5342017-07-20 15:22:39 -0700300 main.log.report( "Scale: " + str( main.Cluster.numCtrls ) + " \tIntent batch: " + str( batchSize ) )
You Wang6d301d42017-04-21 10:49:33 -0700301 main.log.report( "Local latency average:................" + str( aveLocalLatency ) )
302 main.log.report( "Global latency average:................" + str( aveGlobalLatency ) )
303 main.log.report( "Local latency std:................" + str( stdLocalLatency ) )
304 main.log.report( "Global latency std:................" + str( stdGlobalLatency ) )
Chiyu Chengec63bde2016-11-17 18:11:36 -0800305 main.log.report( "________________________________________________________" )
cameron@onlab.us78b89652015-07-08 15:21:03 -0700306
You Wang6d301d42017-04-21 10:49:33 -0700307 if not ( numpy.isnan( aveLocalLatency ) or numpy.isnan( aveGlobalLatency ) ):
YPZhang8742d2e2016-06-16 15:31:58 -0700308 # check if got NaN for result
You Wang6d301d42017-04-21 10:49:33 -0700309 resultsDB = open( main.dbFileName, "a" )
310 resultsDB.write( "'" + main.commit + "'," )
Devin Lim142b5342017-07-20 15:22:39 -0700311 resultsDB.write( str( main.Cluster.numCtrls ) + "," )
You Wang6d301d42017-04-21 10:49:33 -0700312 resultsDB.write( str( batchSize ) + "," )
313 resultsDB.write( str( aveLocalLatency ) + "," )
314 resultsDB.write( str( stdLocalLatency ) + "\n" )
YPZhang8742d2e2016-06-16 15:31:58 -0700315 resultsDB.close()