blob: 45f3741bfead53b70b76f78c5b3862c703badc9f [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2015 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
cameron@onlab.us946d99c2015-07-08 15:34:37 -070021# ScaleOutTemplate -> flowTP
22#
23# CASE1 starts number of nodes specified in param file
24#
25# cameron@onlab.us
26
27import sys
28import os.path
29
30
31class SCPFflowTp1g:
32
33 def __init__( self ):
34 self.default = ''
35
Devin Limd2f199a2017-08-04 12:02:47 -070036 def CASE0( self, main ):
cameron@onlab.us946d99c2015-07-08 15:34:37 -070037 import time
cameron@onlab.us946d99c2015-07-08 15:34:37 -070038 try:
Devin Limd2f199a2017-08-04 12:02:47 -070039 from tests.dependencies.ONOSSetup import ONOSSetup
40 main.testSetUp = ONOSSetup()
41 except ImportError:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070042 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070043 main.cleanAndExit()
Devin Limd2f199a2017-08-04 12:02:47 -070044 main.testSetUp.envSetupDescription()
45 try:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070046 # Load values from params file
Devin Limd2f199a2017-08-04 12:02:47 -070047 cellName = main.params[ 'ENV' ][ 'cellName' ]
48 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
49 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
50 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070051 main.scale = ( main.params[ 'SCALE' ] ).split( "," )
Devin Limd2f199a2017-08-04 12:02:47 -070052 main.flowRuleCfg = main.params[ 'CFG' ][ 'flowRule' ]
Devin Lim9ef93e12017-08-07 17:01:46 -070053 main.neighbor = ( main.params[ 'TEST' ][ 'neighbors' ] ).split( "," )
Devin Limd2f199a2017-08-04 12:02:47 -070054 main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ]
Devin Limccce5c82017-08-17 17:34:12 -070055 isFlowObj = main.params[ 'TEST' ][ 'flowObj' ] == "True"
56 if isFlowObj:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070057 resultFile = main.params[ 'TEST' ][ 'flowObjResultFile' ]
sivachidambaram subramanian1c18af82017-08-03 14:36:18 +053058 else:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070059 resultFile = main.params[ 'TEST' ][ 'flowResultFile' ]
Devin Limd2f199a2017-08-04 12:02:47 -070060 stepResult = main.testSetUp.envSetup()
sivachidambaram subramanian1c18af82017-08-03 14:36:18 +053061 resultsDB = open( str( resultFile ), "w+" )
Devin Limd2f199a2017-08-04 12:02:47 -070062 resultsDB.close()
63 except Exception as e:
64 main.testSetUp.envSetupException( e )
65 main.testSetUp.evnSetupConclusion( stepResult )
66 main.commit = ( main.commit.split( " " ) )[ 1 ]
cameron@onlab.us946d99c2015-07-08 15:34:37 -070067
Devin Limd2f199a2017-08-04 12:02:47 -070068 def CASE1( self, main ):
You Wanga0f6ff62018-01-11 15:46:30 -080069 main.testSetUp.ONOSSetUp( main.Cluster, True, cellName=cellName )
Devin Lim142b5342017-07-20 15:22:39 -070070 main.log.info( "Startup sequence complete" )
71 main.ONOSbench.logReport( main.Cluster.active( 0 ).ipAddress, [ "ERROR", "WARNING", "EXCEPT" ], outputMode="d" )
Chiyu Chengef109502016-11-21 15:51:38 -080072
cameron@onlab.us946d99c2015-07-08 15:34:37 -070073 def CASE2( self, main ):
74 #
sivachidambaram subramanian1c18af82017-08-03 14:36:18 +053075 # This is the flow/flowObjective TP test
cameron@onlab.us946d99c2015-07-08 15:34:37 -070076 #
Jon Hall4ba53f02015-07-29 13:07:41 -070077 import os.path
78 import numpy
cameron@onlab.us946d99c2015-07-08 15:34:37 -070079 import math
Jon Hall4ba53f02015-07-29 13:07:41 -070080 import time
cameron@onlab.us946d99c2015-07-08 15:34:37 -070081 import datetime
82 import traceback
83
Devin Lim142b5342017-07-20 15:22:39 -070084 testCMD = [ 0, 0, 0, 0 ]
85 warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
86 sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
87 switches = int( main.params[ 'TEST' ][ 'switches' ] )
Devin Lim142b5342017-07-20 15:22:39 -070088 testCMD[ 0 ] = main.params[ 'TEST' ][ 'testCMD0' ]
89 testCMD[ 1 ] = main.params[ 'TEST' ][ 'testCMD1' ]
sivachidambaram subramanian1c18af82017-08-03 14:36:18 +053090 testCMD[ 2 ] = main.params[ 'TEST' ][ 'testCMD2' ]
91 testCMD[ 3 ] = main.params[ 'TEST' ][ 'testCMD3' ]
92 flowObjType = main.params[ 'TEST' ][ 'flowObjType' ]
cameron@onlab.us946d99c2015-07-08 15:34:37 -070093 cooldown = main.params[ 'TEST' ][ 'cooldown' ]
94 cellName = main.params[ 'ENV' ][ 'cellName' ]
95 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
96 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
97 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
Devin Lim142b5342017-07-20 15:22:39 -070098 homeDir = os.path.expanduser( '~' )
99 flowRuleBackup = str( main.params[ 'TEST' ][ 'enableFlowRuleStoreBackup' ] )
100 main.log.info( "Flow Rule Backup is set to:" + flowRuleBackup )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700101
Devin Lim142b5342017-07-20 15:22:39 -0700102 servers = str( main.Cluster.numCtrls )
Devin Lim9ef93e12017-08-07 17:01:46 -0700103 neighbors = '0' if main.neighbor.pop( 0 ) == '0' else str( main.Cluster.numCtrls - 1 )
104 main.log.info( "Number of Neighbors: " + neighbors )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700105 ts = time.time()
Devin Lim142b5342017-07-20 15:22:39 -0700106 st = datetime.datetime.fromtimestamp( ts ).strftime( '%Y-%m-%d %H:%M:%S' )
Devin Lim9ef93e12017-08-07 17:01:46 -0700107 main.step( "\tSTARTING TEST" )
108 main.step( "\tLOADING FROM SERVERS: \t" + str( main.Cluster.numCtrls ) )
109 main.step( "\tNEIGHBORS:\t" + neighbors )
110 main.log.info( "=============================================================" )
111 main.log.info( "=============================================================" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700112 # write file to configure nil link
Devin Lim9ef93e12017-08-07 17:01:46 -0700113 ipCSV = ""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700114 for i in range( main.Cluster.maxCtrls ):
Devin Lim9ef93e12017-08-07 17:01:46 -0700115 tempstr = "ip" + str( i + 1 )
116 ipCSV += main.params[ 'CTRL' ][ tempstr ]
117 if i + 1 < main.Cluster.maxCtrls:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700118 ipCSV += ","
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700119
Devin Lim9ef93e12017-08-07 17:01:46 -0700120 main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
121 main.flowRuleCfg,
122 "backupCount 1" )
123 for i in range( 3 ):
Devin Lim142b5342017-07-20 15:22:39 -0700124 main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
Devin Lim9ef93e12017-08-07 17:01:46 -0700125 main.nullProviderCfg,
126 "deviceCount 35" )
127 main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
128 main.nullProviderCfg,
129 "topoShape linear" )
130 main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
131 main.nullProviderCfg,
132 "enabled true" )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700133
Devin Lim9ef93e12017-08-07 17:01:46 -0700134 time.sleep( 5 )
135 main.ONOSbench.handle.sendline( "onos $OC1 summary" )
136 main.ONOSbench.handle.expect( ":~" )
137 check = main.ONOSbench.handle.before
138 main.log.info( "\nStart up check: \n" + check + "\n" )
139 if "SCC(s)=1," in check:
140 main.ONOSbench.handle.sendline( "onos $OC1 balance-masters" )
141 main.ONOSbench.handle.expect( ":~" )
Devin Lim142b5342017-07-20 15:22:39 -0700142 time.sleep( 5 )
Devin Lim9ef93e12017-08-07 17:01:46 -0700143 main.ONOSbench.handle.sendline( "onos $OC1 roles " )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700144 main.ONOSbench.handle.expect( ":~" )
Devin Lim9ef93e12017-08-07 17:01:46 -0700145 main.log.info( "switch masterships:" + str( main.ONOSbench.handle.before ) )
146 break
147 time.sleep( 5 )
148
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700149 # divide flows/flowObjectives
Devin Limccce5c82017-08-17 17:34:12 -0700150 if isFlowObj:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700151 toInstall = "FlowObjectives"
152 installCount = int( main.params[ 'TEST' ][ 'flowObjectives' ] )
153 ifFailed = "FLOW_OBJ_TESTER.PY FAILURE"
154 resultFile = main.params[ 'TEST' ][ 'flowObjResultFile' ]
Devin Lim9ef93e12017-08-07 17:01:46 -0700155 else:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700156 toInstall = "Flows"
157 installCount = int( main.params[ 'TEST' ][ 'flows' ] )
158 ifFailed = "FLOW_TESTER.PY FAILURE"
159 resultFile = main.params[ 'TEST' ][ 'flowResultFile' ]
Devin Lim9ef93e12017-08-07 17:01:46 -0700160 main.log.info( toInstall + " Target = " + str( installCount ) )
161
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700162 installCountPerSwitch = ( installCount * max( int( neighbors ) + 1, int( servers ) ) ) / ( ( int( neighbors ) + 1 ) * int( servers ) * ( switches ) )
Devin Lim9ef93e12017-08-07 17:01:46 -0700163
You Wangcfc98622017-08-11 13:01:00 -0700164 main.log.info( toInstall + " per switch = " + str( installCountPerSwitch ) )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700165 # build list of servers in "$OC1, $OC2...." format
Devin Lim9ef93e12017-08-07 17:01:46 -0700166 serverEnvVars = ""
167 for i in range( int( servers ) ):
168 serverEnvVars += ( "-s " + main.Cluster.active( i ).ipAddress + " " )
169
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700170 data = [ [ "" ] * int( servers ) ] * int( sampleSize )
171 maxes = [ "" ] * int( sampleSize )
Devin Lim9ef93e12017-08-07 17:01:46 -0700172
173 flowCMD = "python3 " + homeDir + "/onos/tools/test/bin/"
Devin Limccce5c82017-08-17 17:34:12 -0700174 if isFlowObj:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700175 flowCMD += testCMD[ 2 ] + " " + str( installCountPerSwitch ) + " " + testCMD[ 1 ]
176 flowCMD += " " + neighbors + " " + testCMD[ 3 ] + " " + str( flowObjType ) + " " + str( serverEnvVars ) + "-j"
Devin Lim9ef93e12017-08-07 17:01:46 -0700177 else:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700178 flowCMD += testCMD[ 0 ] + " " + str( installCountPerSwitch ) + " " + testCMD[ 1 ]
179 flowCMD += " " + neighbors + " " + str( serverEnvVars ) + "-j"
Devin Lim9ef93e12017-08-07 17:01:46 -0700180
181 main.log.info( flowCMD )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700182 # time.sleep( 60 )
Devin Lim9ef93e12017-08-07 17:01:46 -0700183
184 for test in range( 0, warmUp + sampleSize ):
185 if test < warmUp:
186 main.log.info( "Warm up " + str( test + 1 ) + " of " + str( warmUp ) )
187 else:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700188 main.log.info( "====== Test run: " + str( test - warmUp + 1 ) + " ======" )
Devin Lim9ef93e12017-08-07 17:01:46 -0700189
190 main.ONOSbench.handle.sendline( flowCMD )
191 main.ONOSbench.handle.expect( ":~" )
192 rawResult = main.ONOSbench.handle.before
193 main.log.info( "Raw results: \n" + rawResult + "\n" )
194
195 if "failed" in rawResult:
196 main.log.report( ifFailed )
197 main.log.report( " \n" + rawResult + " \n" )
198 for ctrl in main.Cluster.active():
199 main.log.report( "=======================================================" )
200 main.log.report( ctrl.name + "LOG REPORT" )
201 main.ONOSbench.logReport( ctrl.ipAddress, [ "ERROR", "WARNING", "EXCEPT" ], outputMode="d" )
202 main.ONOSbench.handle.sendline( "onos $OC1 flows" )
203 main.ONOSbench.handle.expect( ":~" )
204 main.log.info( main.ONOSbench.handle.before )
205
206 break
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700207 result = [ "" ] * ( main.Cluster.numCtrls )
Devin Lim9ef93e12017-08-07 17:01:46 -0700208
209 rawResult = rawResult.splitlines()
210
211 for node in range( main.Cluster.numCtrls ):
212 for line in rawResult:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700213 # print( "line: " + line )
Devin Lim9ef93e12017-08-07 17:01:46 -0700214 if main.Cluster.active( node ).ipAddress in line and "server" in line:
215 temp = line.split( " " )
216 for word in temp:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700217 # print ( "word: " + word )
Devin Lim9ef93e12017-08-07 17:01:46 -0700218 if "elapsed" in repr( word ):
219 index = temp.index( word ) + 1
220 myParsed = ( temp[ index ] ).replace( ",", "" )
221 myParsed = myParsed.replace( "}", "" )
222 myParsed = int( myParsed )
223 result[ node ] = myParsed
224 main.log.info( main.Cluster.active( node ).ipAddress + " : " + str( myParsed ) )
225 break
226
227 if test >= warmUp:
228 for i in result:
229 if i == "":
230 main.log.error( "Missing data point, critical failure incoming" )
231
232 print result
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700233 maxes[ test - warmUp ] = max( result )
234 main.log.info( "Data collection iteration: " + str( test - warmUp ) + " of " + str( sampleSize ) )
235 main.log.info( "Throughput time: " + str( maxes[ test - warmUp ] ) + "(ms)" )
Devin Lim9ef93e12017-08-07 17:01:46 -0700236
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700237 data[ test - warmUp ] = result
Devin Lim9ef93e12017-08-07 17:01:46 -0700238
239 # wait for flows = 0
240 for checkCount in range( 0, 5 ):
241 time.sleep( 10 )
Devin Lim142b5342017-07-20 15:22:39 -0700242 main.ONOSbench.handle.sendline( "onos $OC1 summary" )
243 main.ONOSbench.handle.expect( ":~" )
Devin Lim9ef93e12017-08-07 17:01:46 -0700244 flowCheck = main.ONOSbench.handle.before
245 if "flows=0," in flowCheck:
246 main.log.info( toInstall + " removed" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700247 break
Jon Hall4ba53f02015-07-29 13:07:41 -0700248 else:
Devin Lim9ef93e12017-08-07 17:01:46 -0700249 for line in flowCheck.splitlines():
250 if "flows=" in line:
251 main.log.info( "Current Summary: " + line )
252 if checkCount == 2:
253 main.log.info( toInstall + " are stuck, moving on " )
254 time.sleep( 5 )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700255
Devin Lim9ef93e12017-08-07 17:01:46 -0700256 main.log.info( "raw data: " + str( data ) )
257 main.log.info( "maxes:" + str( maxes ) )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700258
Devin Lim9ef93e12017-08-07 17:01:46 -0700259 # report data
260 print( "" )
261 main.log.info( "\t Results (measurments are in milliseconds)" )
262 print( "" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700263
Devin Lim9ef93e12017-08-07 17:01:46 -0700264 nodeString = ""
265 for i in range( 1, int( servers ) + 1 ):
266 nodeString += ( "\tNode " + str( i ) )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700267
Devin Lim9ef93e12017-08-07 17:01:46 -0700268 for test in range( 0, sampleSize ):
269 main.log.info( "\t Test iteration " + str( test + 1 ) )
270 main.log.info( "\t------------------" )
271 main.log.info( nodeString )
272 resultString = ""
Jon Hall4ba53f02015-07-29 13:07:41 -0700273
Devin Lim9ef93e12017-08-07 17:01:46 -0700274 for i in range( 0, int( servers ) ):
275 resultString += ( "\t" + str( data[ test ][ i ] ) )
276 main.log.info( resultString )
277 print( "\n" )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700278
Devin Lim9ef93e12017-08-07 17:01:46 -0700279 avgOfMaxes = numpy.mean( maxes )
280 main.log.info( "Average of max value from each test iteration: " + str( avgOfMaxes ) )
Jon Hall4ba53f02015-07-29 13:07:41 -0700281
Devin Lim9ef93e12017-08-07 17:01:46 -0700282 stdOfMaxes = numpy.std( maxes )
283 main.log.info( "Standard Deviation of max values: " + str( stdOfMaxes ) )
284 print( "\n\n" )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700285
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700286 avgTP = int( installCount ) / avgOfMaxes # result in kflows/second
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700287
Devin Lim9ef93e12017-08-07 17:01:46 -0700288 tp = []
289 for i in maxes:
290 tp.append( ( int( installCount ) / i ) )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700291
Devin Lim9ef93e12017-08-07 17:01:46 -0700292 stdTP = numpy.std( tp )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700293
Devin Lim9ef93e12017-08-07 17:01:46 -0700294 main.log.info( "Average thoughput: " + str( avgTP ) + " K" + toInstall + "/second" )
295 main.log.info( "Standard deviation of throughput: " + str( stdTP ) + " K" + toInstall + "/second" )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700296
Devin Lim9ef93e12017-08-07 17:01:46 -0700297 resultsLog = open( str( resultFile ), "a" )
298 resultString = ( "'" + main.commit + "'," )
299 resultString += ( "'1gig'," )
300 resultString += ( str( installCount ) + "," )
301 resultString += ( str( main.Cluster.numCtrls ) + "," )
302 resultString += ( neighbors + "," )
303 resultString += ( str( avgTP ) + "," + str( stdTP ) + "\n" )
304 resultsLog.write( resultString )
305 resultsLog.close()
Jon Hall4ba53f02015-07-29 13:07:41 -0700306
Devin Lim9ef93e12017-08-07 17:01:46 -0700307 main.log.report( "Result line to file: " + resultString )
Jon Hall4ba53f02015-07-29 13:07:41 -0700308
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700309 main.ONOSbench.logReport( main.Cluster.active( 0 ).ipAddress, [ "ERROR", "WARNING", "EXCEPT" ], outputMode="d" )