blob: 7d70d7f51ea0510f0082a45167c9fdff22bc2177 [file] [log] [blame]
cameron@onlab.us78b89652015-07-08 15:21:03 -07001# ScaleOutTemplate
2#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
7import sys
8import os.path
9
10
11class SCPFintentRerouteLat:
12
13 def __init__( self ):
14 self.default = ''
15
16 def CASE1( self, main ):
17
18 import time
19
20 global init
21 try:
22 if type(init) is not bool:
23 init = False
24 except NameError:
25 init = False
26
27 #Load values from params file
28 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
29 gitPull = main.params[ 'GIT' ][ 'autopull' ]
30 cellName = main.params[ 'ENV' ][ 'cellName' ]
31 Apps = main.params[ 'ENV' ][ 'cellApps' ]
32 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
cameron@onlab.usdc9d0692015-07-13 16:02:27 -070033 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
34 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
35 main.maxNodes = int(main.params[ 'max' ])
cameron@onlab.us78b89652015-07-08 15:21:03 -070036 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
37 cellName = main.params[ 'ENV' ][ 'cellName' ]
38
Jon Hall4ba53f02015-07-29 13:07:41 -070039 # -- INIT SECTION, ONLY RUNS ONCE -- #
40 if init == False:
cameron@onlab.us78b89652015-07-08 15:21:03 -070041 init = True
42 global clusterCount #number of nodes running
43 global ONOSIp #list of ONOS IP addresses
Jon Hall4ba53f02015-07-29 13:07:41 -070044 global scale
cameron@onlab.us78b89652015-07-08 15:21:03 -070045 global commit
46
47 clusterCount = 0
48 ONOSIp = [ 0 ]
Jon Hall4ba53f02015-07-29 13:07:41 -070049 scale = (main.params[ 'SCALE' ]).split(",")
cameron@onlab.us78b89652015-07-08 15:21:03 -070050 clusterCount = int(scale[0])
51
Jon Hall4ba53f02015-07-29 13:07:41 -070052 #Populate ONOSIp with ips from params
cameron@onlab.us78b89652015-07-08 15:21:03 -070053 ONOSIp = [0]
54 ONOSIp.extend(main.ONOSbench.getOnosIps())
55
56 print("-----------------" + str(ONOSIp))
57 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
58 if skipMvn != "yes":
59 mvnResult = main.ONOSbench.cleanInstall()
60
61 #git
62 main.step( "Git checkout and pull " + checkoutBranch )
63 if gitPull == 'on':
64 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
65 pullResult = main.ONOSbench.gitPull()
66
67 else:
68 checkoutResult = main.TRUE
69 pullResult = main.TRUE
70 main.log.info( "Skipped git checkout and pull" )
Jon Hall4ba53f02015-07-29 13:07:41 -070071
cameron@onlab.us78b89652015-07-08 15:21:03 -070072 commit = main.ONOSbench.getVersion()
73 commit = (commit.split(" "))[1]
74
suibin584c0702015-07-14 15:57:27 -070075 resultsDB = open("/tmp/IntentRerouteLatDB", "w+")
cameron@onlab.us78b89652015-07-08 15:21:03 -070076 resultsDB.close()
77
78 # -- END OF INIT SECTION --#
Jon Hall4ba53f02015-07-29 13:07:41 -070079
cameron@onlab.us78b89652015-07-08 15:21:03 -070080 clusterCount = int(scale[0])
Jon Hall4ba53f02015-07-29 13:07:41 -070081 scale.remove(scale[0])
82
83 #kill off all onos processes
cameron@onlab.us78b89652015-07-08 15:21:03 -070084 main.log.step("Safety check, killing all ONOS processes")
Jon Hall70b2ff42015-11-17 15:49:44 -080085 main.log.step("before initiating environment setup")
cameron@onlab.usdc9d0692015-07-13 16:02:27 -070086 for node in range(1, main.maxNodes + 1):
cameron@onlab.us78b89652015-07-08 15:21:03 -070087 main.ONOSbench.onosDie(ONOSIp[node])
Jon Hall4ba53f02015-07-29 13:07:41 -070088
cameron@onlab.us78b89652015-07-08 15:21:03 -070089 #Uninstall everywhere
90 main.log.step( "Cleaning Enviornment..." )
cameron@onlab.usdc9d0692015-07-13 16:02:27 -070091 for i in range(1, main.maxNodes + 1):
cameron@onlab.us78b89652015-07-08 15:21:03 -070092 main.log.info(" Uninstalling ONOS " + str(i) )
93 main.ONOSbench.onosUninstall( ONOSIp[i] )
Jon Hall4ba53f02015-07-29 13:07:41 -070094
cameron@onlab.us78b89652015-07-08 15:21:03 -070095 #construct the cell file
96 main.log.info("Creating cell file")
97 cellIp = []
98 for node in range (1, clusterCount + 1):
99 cellIp.append(ONOSIp[node])
cameron@onlab.usdc9d0692015-07-13 16:02:27 -0700100
101 main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), cellIp)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700102
103 main.step( "Set Cell" )
104 main.ONOSbench.setCell(cellName)
Jon Hall4ba53f02015-07-29 13:07:41 -0700105
cameron@onlab.us78b89652015-07-08 15:21:03 -0700106 main.step( "Creating ONOS package" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700107 packageResult = main.ONOSbench.onosPackage()
cameron@onlab.us78b89652015-07-08 15:21:03 -0700108
109 main.step( "verify cells" )
110 verifyCellResult = main.ONOSbench.verifyCell()
Jon Hall4ba53f02015-07-29 13:07:41 -0700111
cameron@onlab.us78b89652015-07-08 15:21:03 -0700112 main.log.report( "Initializing " + str( clusterCount ) + " node cluster." )
113 for node in range(1, clusterCount + 1):
114 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
115 main.ONOSbench.onosInstall( ONOSIp[node])
116
117 for node in range(1, clusterCount + 1):
118 for i in range( 2 ):
119 isup = main.ONOSbench.isup( ONOSIp[node] )
120 if isup:
121 main.log.info("ONOS " + str(node) + " is up\n")
122 break
123 if not isup:
124 main.log.report( "ONOS " + str(node) + " didn't start!" )
125 main.log.info("Startup sequence complete")
Jon Hall4ba53f02015-07-29 13:07:41 -0700126
cameron@onlab.us78b89652015-07-08 15:21:03 -0700127 deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
128 print("Device mastership list: " + str(deviceMastership))
129
130 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.store.flow.impl.NewDistributedFlowRuleStore", "backupEnabled false")
131
132 main.log.step("Setting up null provider")
133 for i in range(3):
134 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "deviceCount 8")
135 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "topoShape reroute")
136 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "enabled true")
137 time.sleep(5)
138 main.ONOSbench.handle.sendline("onos $OC1 summary")
139 main.ONOSbench.handle.expect(":~")
140 x = main.ONOSbench.handle.before
141 if "devices=8" in x and "links=16," in x:
142 break
143
144 index = 1
145 for node in deviceMastership:
146 for attempt in range(0,10):
147 cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)] + """ master" """)
148 main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
149 main.ONOSbench.handle.sendline(cmd)
150 main.ONOSbench.handle.expect(":~")
151 time.sleep(4)
Jon Hall4ba53f02015-07-29 13:07:41 -0700152
153 cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index))
154 main.log.info(cmd)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700155 main.ONOSbench.handle.sendline(cmd)
156 main.ONOSbench.handle.expect(":~")
157 check = main.ONOSbench.handle.before
158 main.log.info("CHECK:\n" + check)
159 if ("master=" + ONOSIp[int(node)]) in check:
160 break
161 index += 1
162
163 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
164
165 def CASE2( self, main ):
Jon Hall4ba53f02015-07-29 13:07:41 -0700166
cameron@onlab.us78b89652015-07-08 15:21:03 -0700167 import time
168 import numpy
169 import datetime
170 #from scipy import stats
171
172 ts = time.time()
cameron@onlab.us78b89652015-07-08 15:21:03 -0700173
174 sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
175 warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
176 intentsList = (main.params[ 'TEST' ][ 'intents' ]).split(",")
177 debug = main.params[ 'TEST' ][ 'debug' ]
178 for i in range(0,len(intentsList)):
179 intentsList[i] = int(intentsList[i])
180
181 timestampMetrics = []
182 if main.params['METRICS']['Submitted'] == "1":
183 timestampMetrics.append("Submitted")
184 if main.params['METRICS']['Installed'] == "1":
185 timestampMetrics.append("Installed")
186 if main.params['METRICS']['Failed'] == "1":
187 timestampMetrics.append("Failed")
188 if main.params['METRICS']['Withdraw'] == "1":
189 timestampMetrics.append("Withdraw")
190 if main.params['METRICS']['Withdrawn'] == "1":
191 timestampMetrics.append("Withdrawn")
192 if debug: main.log.info(timestampMetrics)
193
194 if debug == "True":
195 debug = True
196 else:
197 debug = False
198
199 ingress = "null:0000000000000001"
200 egress = "null:0000000000000007"
201
202 for intents in intentsList:
203 main.log.report("Intent Batch size: " + str(intents) + "\n ")
204 myResult = [["latency", "lastNode"] for x in range(sampleSize)]
205
206 for run in range(0, (warmUp + sampleSize)):
207 if run > warmUp:
208 main.log.info("Starting test iteration " + str(run-warmUp))
209
210 cmd = """onos $OC1 "push-test-intents -i """
211 cmd += ingress + "/0 "
212 cmd += egress + "/0 "
213 cmd += str(intents) +""" 1" """
214 if debug: main.log.info(cmd)
215
216 withdrawCmd = cmd.replace("intents -i", "intents -w ")
217
218 #push-test-intents
219 main.ONOSbench.handle.sendline(cmd)
220 main.ONOSbench.handle.expect(":~")
221 myRawResult = main.ONOSbench.handle.before
222
223 for i in range(0, 40):
224 main.ONOSbench.handle.sendline("onos $OC1 summary")
225 main.ONOSbench.handle.expect(":~")
226 linkCheck = main.ONOSbench.handle.before
227 if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
228 break
229 if i == 39:
230 main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
231 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
232 #main.ONOSbench.logReport(ONOSIp[(clusterCount-1)], ["ERROR", "WARNING", "EXCEPT"], "d")
233 main.ONOSbench.sendline("onos $OC1 summary")
234 main.ONOSbench.sendline("onos $OC1 devices")
Jon Hall4ba53f02015-07-29 13:07:41 -0700235 main.ONOSbench.sendline("onos $OC1 links")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700236 main.ONOSbench.expect(":~")
237 main.log.info(main.ONOSbench.before)
238
239 #collect timestamp from link cut
240 cmd = """onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 down" """
241 if debug: main.log.info("COMMAND: " + str(cmd))
242 main.ONOSbench.handle.sendline(cmd)
243 main.ONOSbench.handle.expect(":~")
244
245 cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
246 for i in range(0,10):
247 main.ONOSbench.handle.sendline(cmd)
248 time.sleep(2)
249 main.ONOSbench.handle.expect(":~")
250 raw = main.ONOSbench.handle.before
251 #if "NullLinkProvider" in raw and "links=14" in raw:
252 if "links=14" in raw:
253 break
254 if i >= 9:
255 main.log.error("Expected output not being recieved... continuing")
256 main.log.info(raw)
257 break
258 time.sleep(2)
259
GlennRC2b28a412015-10-14 10:28:50 -0700260 if debug: main.log.debug("raw: " + raw)
261
GlennRC9abad1b2015-10-15 11:47:12 -0700262 temp = raw.splitlines()
GlennRC2b28a412015-10-14 10:28:50 -0700263
264 if debug: main.log.debug("temp (after splitlines): " + str(temp))
265
GlennRC9abad1b2015-10-15 11:47:12 -0700266 # Since the string is deterministic the date is always the 3rd element.
267 # However, if the data were grepping for in the onos log changes then this will
268 # not work. This is why we print out the raw and temp string so we can visually
269 # check if everything is in the correct order. temp should like this:
270 # temp = ['/onos$ onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep Top ',
271 # 'ologyManager| tail -1', '2015-10-15 12:03:33,736 ... ]
272 temp = temp[2]
cameron@onlab.us78b89652015-07-08 15:21:03 -0700273
GlennRC2b28a412015-10-14 10:28:50 -0700274 if debug: main.log.debug("temp (checking for date): " + str(temp))
275
GlennRC9abad1b2015-10-15 11:47:12 -0700276 cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1]
GlennRC2b28a412015-10-14 10:28:50 -0700277
Jon Hall4ba53f02015-07-29 13:07:41 -0700278 if debug: main.log.info("Cut timestamp: " + cutTimestamp)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700279
280 #validate link count and flow count
281 for i in range(0, 40):
282 main.ONOSbench.handle.sendline("onos $OC1 summary")
283 main.ONOSbench.handle.expect(":~")
284 linkCheck = main.ONOSbench.handle.before
285 #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
286 if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
287 break
288 if i == 39:
289 main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
290 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
291
Jon Hall4ba53f02015-07-29 13:07:41 -0700292 time.sleep(5) #trying to avoid negative values
cameron@onlab.us78b89652015-07-08 15:21:03 -0700293
294 #intents events metrics installed timestamp
295 IEMtimestamps = [0]*(clusterCount + 1)
296 installedTemp = [0]*(clusterCount + 1)
297 for node in range(1, clusterCount +1):
298 cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
299 raw = ""
300 while "epoch)" not in raw:
301 main.ONOSbench.handle.sendline(cmd)
302 main.ONOSbench.handle.expect(":~")
303 raw = main.ONOSbench.handle.before
304
305 print(raw)
306
307 intentsTimestamps = {}
308 rawTimestamps = raw.splitlines()
309 for line in rawTimestamps:
310 if "Timestamp" in line and "grep" not in line:
311 metricKey = (line.split(" "))[1]
312 metricTimestamp = (line.split(" ")[len(line.split(" ")) -1]).replace("epoch)=","")
313 metricTimestamp = float(metricTimestamp)
314 metricTimestamp = numpy.divide(metricTimestamp, 1000)
315 if debug: main.log.info(repr(metricTimestamp))
316 intentsTimestamps[metricKey] = metricTimestamp
317 if metricKey == "Installed":
318 installedTemp[node] = metricTimestamp
319
320 main.log.info("Node: " + str(node) + " Timestamps: " + str(intentsTimestamps))
321 IEMtimestamps[node] = intentsTimestamps
322
323 myMax = max(installedTemp)
324 indexOfMax = installedTemp.index(myMax)
325
326 #number crunch
327 for metric in timestampMetrics: #this is where we sould add support for computing other timestamp metrics
328 if metric == "Installed":
329 if run >= warmUp:
330 main.log.report("link cut timestamp: " + cutTimestamp)
331 #readableInstalledTimestamp = str(intentsTimestamps["Installed"])
332 readableInstalledTimestamp = str(myMax)
333
334 #main.log.report("Intent Installed timestamp: " + str(intentsTimestamps["Installed"]))
335 main.log.report("Intent Installed timestamp: " + str(myMax))
336
337 cutEpoch = time.mktime(time.strptime(cutTimestamp, "%Y-%m-%d %H:%M:%S,%f"))
338 if debug: main.log.info("cutEpoch=" + str(cutEpoch))
339 #rerouteLatency = float(intentsTimestamps["Installed"] - cutEpoch)
340 rerouteLatency = float(myMax - cutEpoch)
341
342 rerouteLatency = numpy.divide(rerouteLatency, 1000)
343 main.log.report("Reroute latency:" + str(rerouteLatency) + " (seconds)\n ")
344 myResult[run-warmUp][0] = rerouteLatency
345 myResult[run-warmUp][1] = indexOfMax
346 if debug: main.log.info("Latency: " + str(myResult[run-warmUp][0]))
347 if debug: main.log.info("last node: " + str(myResult[run-warmUp][1]))
348
349 cmd = """ onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 up" """
350 if debug: main.log.info(cmd)
351 main.ONOSbench.handle.sendline(cmd)
352 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700353
cameron@onlab.us78b89652015-07-08 15:21:03 -0700354 #wait for intent withdraw
355 main.ONOSbench.handle.sendline(withdrawCmd)
Jon Hall4ba53f02015-07-29 13:07:41 -0700356 main.log.info(withdrawCmd)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700357 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700358 if debug: main.log.info(main.ONOSbench.handle.before)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700359 main.ONOSbench.handle.sendline("onos $OC1 intents|grep WITHDRAWN|wc -l")
360 main.ONOSbench.handle.expect(":~")
361 intentWithdrawCheck = main.ONOSbench.handle.before
362 if (str(intents)) in intentWithdrawCheck:
363 main.log.info("intents withdrawn")
364 if debug: main.log.info(intentWithdrawCheck)
365
366 # wait for links to be reestablished
367 for i in range(0, 10):
368 main.ONOSbench.handle.sendline("onos $OC1 summary")
369 main.ONOSbench.handle.expect(":~")
370 linkCheck = main.ONOSbench.handle.before
371 if "links=16," in linkCheck:
372 break
373 time.sleep(1)
374 if i == 9:
375 main.log.info("Links Failed to reconnect, next iteration of data invalid." + linkCheck)
376
377 if run < warmUp:
378 main.log.info("Warm up run " + str(run+1) + " completed")
379
380 if debug: main.log.info(myResult)
381 latTemp = []
382 nodeTemp = []
383 for i in myResult:
384 latTemp.append(i[0])
385 nodeTemp.append(i[1])
Jon Hall4ba53f02015-07-29 13:07:41 -0700386
cameron@onlab.us78b89652015-07-08 15:21:03 -0700387 mode = {}
388 for i in nodeTemp:
389 if i in mode:
390 mode[i] += 1
391 else:
392 mode[i] = 1
393
394 for i in mode.keys():
395 if mode[i] == max(mode.values()):
396 nodeMode = i
397
398 average = numpy.average(latTemp)
399 stdDev = numpy.std(latTemp)
400
401 average = numpy.multiply(average, 1000)
402 stdDev = numpy.multiply(stdDev, 1000)
403
404 main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents))
405 main.log.report("Latency average:................" + str(average))
406 main.log.report("Latency standard deviation:....." + str(stdDev))
407 main.log.report("Mode of last node to respond:..." + str(nodeMode))
408 main.log.report("________________________________________________________")
409
suibine1e58772015-07-15 09:52:59 -0700410 resultsDB = open("/tmp/IntentRerouteLatDB", "a")
Jon Hall4ba53f02015-07-29 13:07:41 -0700411 resultsDB.write("'" + commit + "',")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700412 resultsDB.write(str(clusterCount) + ",")
413 resultsDB.write(str(intents) + ",")
414 resultsDB.write(str(average) + ",")
415 resultsDB.write(str(stdDev) + "\n")
416 resultsDB.close()
417
418 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
419