blob: 7a1f7382cbb3e3adbe384079c95b0674358e1212 [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
2Copyright 2015 Open Networking Foundation (ONF)
3
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
11 (at your option) any later version.
12
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
21
cameron@onlab.us946d99c2015-07-08 15:34:37 -070022# ScaleOutTemplate -> flowTP
23#
24# CASE1 starts number of nodes specified in param file
25#
26# cameron@onlab.us
27
28import sys
29import os.path
30
31
32class SCPFflowTp1g:
33
34 def __init__( self ):
35 self.default = ''
36
Devin Limd2f199a2017-08-04 12:02:47 -070037 def CASE0( self, main ):
cameron@onlab.us946d99c2015-07-08 15:34:37 -070038 import time
cameron@onlab.us946d99c2015-07-08 15:34:37 -070039 try:
Devin Limd2f199a2017-08-04 12:02:47 -070040 from tests.dependencies.ONOSSetup import ONOSSetup
41 main.testSetUp = ONOSSetup()
42 except ImportError:
43 main.log.error("ONOSSetup not found. exiting the test")
Devin Lim44075962017-08-11 10:56:37 -070044 main.cleanAndExit()
Devin Limd2f199a2017-08-04 12:02:47 -070045 main.testSetUp.envSetupDescription()
46 try:
47 #Load values from params file
48 cellName = main.params[ 'ENV' ][ 'cellName' ]
49 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
50 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
51 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
52 main.scale = ( main.params[ 'SCALE' ] ).split( "," )
53 main.flowRuleCfg = main.params[ 'CFG' ][ 'flowRule' ]
Devin Lim9ef93e12017-08-07 17:01:46 -070054 main.neighbor = ( main.params[ 'TEST' ][ 'neighbors' ] ).split( "," )
Devin Limd2f199a2017-08-04 12:02:47 -070055 main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ]
sivachidambaram subramanian1c18af82017-08-03 14:36:18 +053056 isFlowObj = main.params[ 'TEST' ][ 'flowObj' ]
57 if isFlowObj == 'true':
58 resultFile = main.params[ 'TEST' ][ 'flowObjResultFile' ]
59 else:
60 resultFile = main.params[ 'TEST' ][ 'flowResultFile' ]
Devin Limd2f199a2017-08-04 12:02:47 -070061 stepResult = main.testSetUp.envSetup()
sivachidambaram subramanian1c18af82017-08-03 14:36:18 +053062 resultsDB = open( str( resultFile ), "w+" )
Devin Limd2f199a2017-08-04 12:02:47 -070063 resultsDB.close()
64 except Exception as e:
65 main.testSetUp.envSetupException( e )
66 main.testSetUp.evnSetupConclusion( stepResult )
67 main.commit = ( main.commit.split( " " ) )[ 1 ]
cameron@onlab.us946d99c2015-07-08 15:34:37 -070068
Devin Limd2f199a2017-08-04 12:02:47 -070069 def CASE1( self, main ):
Devin Lim142b5342017-07-20 15:22:39 -070070 main.testSetUp.ONOSSetUp( "localhost", main.Cluster, True, cellName=cellName )
Devin Lim142b5342017-07-20 15:22:39 -070071 main.log.info( "Startup sequence complete" )
72 main.ONOSbench.logReport( main.Cluster.active( 0 ).ipAddress, [ "ERROR", "WARNING", "EXCEPT" ], outputMode="d" )
Chiyu Chengef109502016-11-21 15:51:38 -080073
cameron@onlab.us946d99c2015-07-08 15:34:37 -070074 def CASE2( self, main ):
75 #
sivachidambaram subramanian1c18af82017-08-03 14:36:18 +053076 # This is the flow/flowObjective TP test
cameron@onlab.us946d99c2015-07-08 15:34:37 -070077 #
Jon Hall4ba53f02015-07-29 13:07:41 -070078 import os.path
79 import numpy
cameron@onlab.us946d99c2015-07-08 15:34:37 -070080 import math
Jon Hall4ba53f02015-07-29 13:07:41 -070081 import time
cameron@onlab.us946d99c2015-07-08 15:34:37 -070082 import datetime
83 import traceback
84
Devin Lim142b5342017-07-20 15:22:39 -070085 testCMD = [ 0, 0, 0, 0 ]
86 warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
87 sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
88 switches = int( main.params[ 'TEST' ][ 'switches' ] )
Devin Lim142b5342017-07-20 15:22:39 -070089 testCMD[ 0 ] = main.params[ 'TEST' ][ 'testCMD0' ]
90 testCMD[ 1 ] = main.params[ 'TEST' ][ 'testCMD1' ]
sivachidambaram subramanian1c18af82017-08-03 14:36:18 +053091 testCMD[ 2 ] = main.params[ 'TEST' ][ 'testCMD2' ]
92 testCMD[ 3 ] = main.params[ 'TEST' ][ 'testCMD3' ]
93 flowObjType = main.params[ 'TEST' ][ 'flowObjType' ]
94 isFlowObj = main.params[ 'TEST' ][ 'flowObj' ]
cameron@onlab.us946d99c2015-07-08 15:34:37 -070095 cooldown = main.params[ 'TEST' ][ 'cooldown' ]
96 cellName = main.params[ 'ENV' ][ 'cellName' ]
97 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
98 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
99 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
Devin Lim142b5342017-07-20 15:22:39 -0700100 homeDir = os.path.expanduser( '~' )
101 flowRuleBackup = str( main.params[ 'TEST' ][ 'enableFlowRuleStoreBackup' ] )
102 main.log.info( "Flow Rule Backup is set to:" + flowRuleBackup )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700103
Devin Lim142b5342017-07-20 15:22:39 -0700104 servers = str( main.Cluster.numCtrls )
Devin Lim9ef93e12017-08-07 17:01:46 -0700105 neighbors = '0' if main.neighbor.pop( 0 ) == '0' else str( main.Cluster.numCtrls - 1 )
106 main.log.info( "Number of Neighbors: " + neighbors )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700107 ts = time.time()
Devin Lim142b5342017-07-20 15:22:39 -0700108 st = datetime.datetime.fromtimestamp( ts ).strftime( '%Y-%m-%d %H:%M:%S' )
Devin Lim9ef93e12017-08-07 17:01:46 -0700109 main.step( "\tSTARTING TEST" )
110 main.step( "\tLOADING FROM SERVERS: \t" + str( main.Cluster.numCtrls ) )
111 main.step( "\tNEIGHBORS:\t" + neighbors )
112 main.log.info( "=============================================================" )
113 main.log.info( "=============================================================" )
114 #write file to configure nil link
115 ipCSV = ""
116 for i in range ( main.Cluster.maxCtrls ):
117 tempstr = "ip" + str( i + 1 )
118 ipCSV += main.params[ 'CTRL' ][ tempstr ]
119 if i + 1 < main.Cluster.maxCtrls:
120 ipCSV +=","
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700121
Devin Lim9ef93e12017-08-07 17:01:46 -0700122 main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
123 main.flowRuleCfg,
124 "backupCount 1" )
125 for i in range( 3 ):
Devin Lim142b5342017-07-20 15:22:39 -0700126 main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
Devin Lim9ef93e12017-08-07 17:01:46 -0700127 main.nullProviderCfg,
128 "deviceCount 35" )
129 main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
130 main.nullProviderCfg,
131 "topoShape linear" )
132 main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
133 main.nullProviderCfg,
134 "enabled true" )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700135
Devin Lim9ef93e12017-08-07 17:01:46 -0700136 time.sleep( 5 )
137 main.ONOSbench.handle.sendline( "onos $OC1 summary" )
138 main.ONOSbench.handle.expect( ":~" )
139 check = main.ONOSbench.handle.before
140 main.log.info( "\nStart up check: \n" + check + "\n" )
141 if "SCC(s)=1," in check:
142 main.ONOSbench.handle.sendline( "onos $OC1 balance-masters" )
143 main.ONOSbench.handle.expect( ":~" )
Devin Lim142b5342017-07-20 15:22:39 -0700144 time.sleep( 5 )
Devin Lim9ef93e12017-08-07 17:01:46 -0700145 main.ONOSbench.handle.sendline( "onos $OC1 roles " )
146 main.ONOSbench.handle.expect ( ":~" )
147 main.log.info( "switch masterships:" + str( main.ONOSbench.handle.before ) )
148 break
149 time.sleep( 5 )
150
151 #divide flows/flowObjectives
152 if isFlowObj == 'true':
153 toInstall = "FlowObjectives"
154 installCount = int( main.params[ 'TEST' ][ 'flowObjectives' ] )
155 ifFailed = "FLOW_OBJ_TESTER.PY FAILURE"
156 resultFile = main.params[ 'TEST' ][ 'flowObjResultFile' ]
157 else:
158 toInstall = "Flows"
159 installCount = int( main.params[ 'TEST' ][ 'flows' ] )
160 ifFailed = "FLOW_TESTER.PY FAILURE"
161 resultFile = main.params[ 'TEST' ][ 'flowResultFile' ]
162 main.log.info( toInstall + " Target = " + str( installCount ) )
163
You Wangcfc98622017-08-11 13:01:00 -0700164 installCountPerSwitch = ( installCount *max( int ( neighbors ) + 1, int( servers ) ) )/( ( int( neighbors ) + 1 )*int( servers )*( switches ) )
Devin Lim9ef93e12017-08-07 17:01:46 -0700165
You Wangcfc98622017-08-11 13:01:00 -0700166 main.log.info( toInstall + " per switch = " + str( installCountPerSwitch ) )
Devin Lim9ef93e12017-08-07 17:01:46 -0700167 #build list of servers in "$OC1, $OC2...." format
168 serverEnvVars = ""
169 for i in range( int( servers ) ):
170 serverEnvVars += ( "-s " + main.Cluster.active( i ).ipAddress + " " )
171
172 data = [ [ "" ]*int( servers ) ]*int( sampleSize )
173 maxes = [ "" ]*int( sampleSize )
174
175 flowCMD = "python3 " + homeDir + "/onos/tools/test/bin/"
176 if isFlowObj == 'true':
You Wangcfc98622017-08-11 13:01:00 -0700177 flowCMD += testCMD[ 2 ] + " " + str( installCountPerSwitch ) + " " + testCMD[ 1 ]
Devin Lim9ef93e12017-08-07 17:01:46 -0700178 flowCMD += " " + neighbors + " " + testCMD[ 3 ] + " " + str( flowObjType ) + " " + str( serverEnvVars ) + "-j"
179 else:
You Wangcfc98622017-08-11 13:01:00 -0700180 flowCMD += testCMD[ 0 ] + " " + str( installCountPerSwitch ) + " " + testCMD[ 1 ]
Devin Lim9ef93e12017-08-07 17:01:46 -0700181 flowCMD += " " + neighbors + " " + str( serverEnvVars ) + "-j"
182
183 main.log.info( flowCMD )
184 #time.sleep( 60 )
185
186 for test in range( 0, warmUp + sampleSize ):
187 if test < warmUp:
188 main.log.info( "Warm up " + str( test + 1 ) + " of " + str( warmUp ) )
189 else:
190 main.log.info( "====== Test run: " + str( test-warmUp+1 ) + " ======" )
191
192 main.ONOSbench.handle.sendline( flowCMD )
193 main.ONOSbench.handle.expect( ":~" )
194 rawResult = main.ONOSbench.handle.before
195 main.log.info( "Raw results: \n" + rawResult + "\n" )
196
197 if "failed" in rawResult:
198 main.log.report( ifFailed )
199 main.log.report( " \n" + rawResult + " \n" )
200 for ctrl in main.Cluster.active():
201 main.log.report( "=======================================================" )
202 main.log.report( ctrl.name + "LOG REPORT" )
203 main.ONOSbench.logReport( ctrl.ipAddress, [ "ERROR", "WARNING", "EXCEPT" ], outputMode="d" )
204 main.ONOSbench.handle.sendline( "onos $OC1 flows" )
205 main.ONOSbench.handle.expect( ":~" )
206 main.log.info( main.ONOSbench.handle.before )
207
208 break
209 result = [ "" ]*( main.Cluster.numCtrls )
210
211 rawResult = rawResult.splitlines()
212
213 for node in range( main.Cluster.numCtrls ):
214 for line in rawResult:
215 #print( "line: " + line )
216 if main.Cluster.active( node ).ipAddress in line and "server" in line:
217 temp = line.split( " " )
218 for word in temp:
219 #print ( "word: " + word )
220 if "elapsed" in repr( word ):
221 index = temp.index( word ) + 1
222 myParsed = ( temp[ index ] ).replace( ",", "" )
223 myParsed = myParsed.replace( "}", "" )
224 myParsed = int( myParsed )
225 result[ node ] = myParsed
226 main.log.info( main.Cluster.active( node ).ipAddress + " : " + str( myParsed ) )
227 break
228
229 if test >= warmUp:
230 for i in result:
231 if i == "":
232 main.log.error( "Missing data point, critical failure incoming" )
233
234 print result
235 maxes[ test-warmUp ] = max( result )
236 main.log.info( "Data collection iteration: " + str( test-warmUp ) + " of " + str( sampleSize ) )
237 main.log.info( "Throughput time: " + str( maxes[ test-warmUp ] ) + "(ms)" )
238
239 data[ test-warmUp ] = result
240
241 # wait for flows = 0
242 for checkCount in range( 0, 5 ):
243 time.sleep( 10 )
Devin Lim142b5342017-07-20 15:22:39 -0700244 main.ONOSbench.handle.sendline( "onos $OC1 summary" )
245 main.ONOSbench.handle.expect( ":~" )
Devin Lim9ef93e12017-08-07 17:01:46 -0700246 flowCheck = main.ONOSbench.handle.before
247 if "flows=0," in flowCheck:
248 main.log.info( toInstall + " removed" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700249 break
Jon Hall4ba53f02015-07-29 13:07:41 -0700250 else:
Devin Lim9ef93e12017-08-07 17:01:46 -0700251 for line in flowCheck.splitlines():
252 if "flows=" in line:
253 main.log.info( "Current Summary: " + line )
254 if checkCount == 2:
255 main.log.info( toInstall + " are stuck, moving on " )
256 time.sleep( 5 )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700257
Devin Lim9ef93e12017-08-07 17:01:46 -0700258 main.log.info( "raw data: " + str( data ) )
259 main.log.info( "maxes:" + str( maxes ) )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700260
261
Devin Lim9ef93e12017-08-07 17:01:46 -0700262 # report data
263 print( "" )
264 main.log.info( "\t Results (measurments are in milliseconds)" )
265 print( "" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700266
Devin Lim9ef93e12017-08-07 17:01:46 -0700267 nodeString = ""
268 for i in range( 1, int( servers ) + 1 ):
269 nodeString += ( "\tNode " + str( i ) )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700270
Devin Lim9ef93e12017-08-07 17:01:46 -0700271 for test in range( 0, sampleSize ):
272 main.log.info( "\t Test iteration " + str( test + 1 ) )
273 main.log.info( "\t------------------" )
274 main.log.info( nodeString )
275 resultString = ""
Jon Hall4ba53f02015-07-29 13:07:41 -0700276
Devin Lim9ef93e12017-08-07 17:01:46 -0700277 for i in range( 0, int( servers ) ):
278 resultString += ( "\t" + str( data[ test ][ i ] ) )
279 main.log.info( resultString )
280 print( "\n" )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700281
Devin Lim9ef93e12017-08-07 17:01:46 -0700282 avgOfMaxes = numpy.mean( maxes )
283 main.log.info( "Average of max value from each test iteration: " + str( avgOfMaxes ) )
Jon Hall4ba53f02015-07-29 13:07:41 -0700284
Devin Lim9ef93e12017-08-07 17:01:46 -0700285 stdOfMaxes = numpy.std( maxes )
286 main.log.info( "Standard Deviation of max values: " + str( stdOfMaxes ) )
287 print( "\n\n" )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700288
Devin Lim9ef93e12017-08-07 17:01:46 -0700289 avgTP = int( installCount ) / avgOfMaxes #result in kflows/second
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700290
Devin Lim9ef93e12017-08-07 17:01:46 -0700291 tp = []
292 for i in maxes:
293 tp.append( ( int( installCount ) / i ) )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700294
Devin Lim9ef93e12017-08-07 17:01:46 -0700295 stdTP = numpy.std( tp )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700296
Devin Lim9ef93e12017-08-07 17:01:46 -0700297 main.log.info( "Average thoughput: " + str( avgTP ) + " K" + toInstall + "/second" )
298 main.log.info( "Standard deviation of throughput: " + str( stdTP ) + " K" + toInstall + "/second" )
cameron@onlab.us946d99c2015-07-08 15:34:37 -0700299
Devin Lim9ef93e12017-08-07 17:01:46 -0700300 resultsLog = open( str( resultFile ), "a" )
301 resultString = ( "'" + main.commit + "'," )
302 resultString += ( "'1gig'," )
303 resultString += ( str( installCount ) + "," )
304 resultString += ( str( main.Cluster.numCtrls ) + "," )
305 resultString += ( neighbors + "," )
306 resultString += ( str( avgTP ) + "," + str( stdTP ) + "\n" )
307 resultsLog.write( resultString )
308 resultsLog.close()
Jon Hall4ba53f02015-07-29 13:07:41 -0700309
Devin Lim9ef93e12017-08-07 17:01:46 -0700310 main.log.report( "Result line to file: " + resultString )
Jon Hall4ba53f02015-07-29 13:07:41 -0700311
Devin Lima4f95bc2017-08-11 11:13:03 -0700312 main.ONOSbench.logReport( main.Cluster.active( 0 ).ipAddress, [ "ERROR", "WARNING", "EXCEPT" ], outputMode="d" )