Gitiles
Code Review Sign In
gerrit.onosproject.org / OnosSystemTest / d48a68c13228bc4ada5b706362eb40970bee4519 / . / TestON / tests / SCPFintentRerouteLat / SCPFintentRerouteLat.py
blob: cc8892ee0e7ea756945c2a8b5e57797566c135ad [file] [log] [blame]
cameron@onlab.us78b89652015-07-08 15:21:03 -0700[diff] [blame]1# ScaleOutTemplate
2#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
7import sys
8import os.path
9
10
11class SCPFintentRerouteLat:
12
13 def __init__( self ):
14 self.default = ''
15
16 def CASE1( self, main ):
17
18 import time
19
20 global init
21 try:
22 if type(init) is not bool:
23 init = False
24 except NameError:
25 init = False
26
27 #Load values from params file
28 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
29 gitPull = main.params[ 'GIT' ][ 'autopull' ]
30 cellName = main.params[ 'ENV' ][ 'cellName' ]
31 Apps = main.params[ 'ENV' ][ 'cellApps' ]
32 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
33 maxNodes = int(main.params[ 'availableNodes' ])
34 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
35 cellName = main.params[ 'ENV' ][ 'cellName' ]
36
37 # -- INIT SECTION, ONLY RUNS ONCE -- #
38 if init == False:
39 init = True
40 global clusterCount #number of nodes running
41 global ONOSIp #list of ONOS IP addresses
42 global scale
43 global commit
44
45 clusterCount = 0
46 ONOSIp = [ 0 ]
47 scale = (main.params[ 'SCALE' ]).split(",")
48 clusterCount = int(scale[0])
49
50 #Populate ONOSIp with ips from params
51 ONOSIp = [0]
52 ONOSIp.extend(main.ONOSbench.getOnosIps())
53
54 print("-----------------" + str(ONOSIp))
55 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
56 if skipMvn != "yes":
57 mvnResult = main.ONOSbench.cleanInstall()
58
59 #git
60 main.step( "Git checkout and pull " + checkoutBranch )
61 if gitPull == 'on':
62 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
63 pullResult = main.ONOSbench.gitPull()
64
65 else:
66 checkoutResult = main.TRUE
67 pullResult = main.TRUE
68 main.log.info( "Skipped git checkout and pull" )
69
70 commit = main.ONOSbench.getVersion()
71 commit = (commit.split(" "))[1]
72
73 resultsDB = open("IntentRerouteLatDB", "w+")
74 resultsDB.close()
75
76 # -- END OF INIT SECTION --#
77
78 clusterCount = int(scale[0])
79 scale.remove(scale[0])
80
81 MN1Ip = ONOSIp[len(ONOSIp)-1]
82 BENCHIp = ONOSIp[len(ONOSIp)-2]
83
84 #kill off all onos processes
85 main.log.step("Safety check, killing all ONOS processes")
86 main.log.step("before initiating enviornment setup")
87 for node in range(1, maxNodes + 1):
88 main.ONOSbench.onosDie(ONOSIp[node])
89
90 #Uninstall everywhere
91 main.log.step( "Cleaning Enviornment..." )
92 for i in range(1, maxNodes + 1):
93 main.log.info(" Uninstalling ONOS " + str(i) )
94 main.ONOSbench.onosUninstall( ONOSIp[i] )
95
96 #construct the cell file
97 main.log.info("Creating cell file")
98 cellIp = []
99 for node in range (1, clusterCount + 1):
100 cellIp.append(ONOSIp[node])
101
102 main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), *cellIp)
103
104 main.step( "Set Cell" )
105 main.ONOSbench.setCell(cellName)
106
107 main.step( "Creating ONOS package" )
108 packageResult = main.ONOSbench.onosPackage()
109
110 main.step( "verify cells" )
111 verifyCellResult = main.ONOSbench.verifyCell()
112
113 main.log.report( "Initializing " + str( clusterCount ) + " node cluster." )
114 for node in range(1, clusterCount + 1):
115 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
116 main.ONOSbench.onosInstall( ONOSIp[node])
117
118 for node in range(1, clusterCount + 1):
119 for i in range( 2 ):
120 isup = main.ONOSbench.isup( ONOSIp[node] )
121 if isup:
122 main.log.info("ONOS " + str(node) + " is up\n")
123 break
124 if not isup:
125 main.log.report( "ONOS " + str(node) + " didn't start!" )
126 main.log.info("Startup sequence complete")
127
128 deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
129 print("Device mastership list: " + str(deviceMastership))
130
131 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.store.flow.impl.NewDistributedFlowRuleStore", "backupEnabled false")
132
133 main.log.step("Setting up null provider")
134 for i in range(3):
135 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "deviceCount 8")
136 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "topoShape reroute")
137 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "enabled true")
138 time.sleep(5)
139 main.ONOSbench.handle.sendline("onos $OC1 summary")
140 main.ONOSbench.handle.expect(":~")
141 x = main.ONOSbench.handle.before
142 if "devices=8" in x and "links=16," in x:
143 break
144
145 index = 1
146 for node in deviceMastership:
147 for attempt in range(0,10):
148 cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)] + """ master" """)
149 main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
150 main.ONOSbench.handle.sendline(cmd)
151 main.ONOSbench.handle.expect(":~")
152 time.sleep(4)
153
154 cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index))
155 main.log.info(cmd)
156 main.ONOSbench.handle.sendline(cmd)
157 main.ONOSbench.handle.expect(":~")
158 check = main.ONOSbench.handle.before
159 main.log.info("CHECK:\n" + check)
160 if ("master=" + ONOSIp[int(node)]) in check:
161 break
162 index += 1
163
164 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
165
166 def CASE2( self, main ):
167
168 import time
169 import numpy
170 import datetime
171 #from scipy import stats
172
173 ts = time.time()
174 date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d')
175
176 sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
177 warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
178 intentsList = (main.params[ 'TEST' ][ 'intents' ]).split(",")
179 debug = main.params[ 'TEST' ][ 'debug' ]
180 for i in range(0,len(intentsList)):
181 intentsList[i] = int(intentsList[i])
182
183 timestampMetrics = []
184 if main.params['METRICS']['Submitted'] == "1":
185 timestampMetrics.append("Submitted")
186 if main.params['METRICS']['Installed'] == "1":
187 timestampMetrics.append("Installed")
188 if main.params['METRICS']['Failed'] == "1":
189 timestampMetrics.append("Failed")
190 if main.params['METRICS']['Withdraw'] == "1":
191 timestampMetrics.append("Withdraw")
192 if main.params['METRICS']['Withdrawn'] == "1":
193 timestampMetrics.append("Withdrawn")
194 if debug: main.log.info(timestampMetrics)
195
196 if debug == "True":
197 debug = True
198 else:
199 debug = False
200
201 ingress = "null:0000000000000001"
202 egress = "null:0000000000000007"
203
204 for intents in intentsList:
205 main.log.report("Intent Batch size: " + str(intents) + "\n ")
206 myResult = [["latency", "lastNode"] for x in range(sampleSize)]
207
208 for run in range(0, (warmUp + sampleSize)):
209 if run > warmUp:
210 main.log.info("Starting test iteration " + str(run-warmUp))
211
212 cmd = """onos $OC1 "push-test-intents -i """
213 cmd += ingress + "/0 "
214 cmd += egress + "/0 "
215 cmd += str(intents) +""" 1" """
216 if debug: main.log.info(cmd)
217
218 withdrawCmd = cmd.replace("intents -i", "intents -w ")
219
220 #push-test-intents
221 main.ONOSbench.handle.sendline(cmd)
222 main.ONOSbench.handle.expect(":~")
223 myRawResult = main.ONOSbench.handle.before
224
225 for i in range(0, 40):
226 main.ONOSbench.handle.sendline("onos $OC1 summary")
227 main.ONOSbench.handle.expect(":~")
228 linkCheck = main.ONOSbench.handle.before
229 if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
230 break
231 if i == 39:
232 main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
233 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
234 #main.ONOSbench.logReport(ONOSIp[(clusterCount-1)], ["ERROR", "WARNING", "EXCEPT"], "d")
235 main.ONOSbench.sendline("onos $OC1 summary")
236 main.ONOSbench.sendline("onos $OC1 devices")
237 main.ONOSbench.sendline("onos $OC1 links")
238 main.ONOSbench.expect(":~")
239 main.log.info(main.ONOSbench.before)
240
241 #collect timestamp from link cut
242 cmd = """onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 down" """
243 if debug: main.log.info("COMMAND: " + str(cmd))
244 main.ONOSbench.handle.sendline(cmd)
245 main.ONOSbench.handle.expect(":~")
246
247 cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
248 for i in range(0,10):
249 main.ONOSbench.handle.sendline(cmd)
250 time.sleep(2)
251 main.ONOSbench.handle.expect(":~")
252 raw = main.ONOSbench.handle.before
253 #if "NullLinkProvider" in raw and "links=14" in raw:
254 if "links=14" in raw:
255 break
256 if i >= 9:
257 main.log.error("Expected output not being recieved... continuing")
258 main.log.info(raw)
259 break
260 time.sleep(2)
261
262 temp = raw.splitlines()
263 for line in temp:
264 if str(date) in line:
265 temp = line
266 break
267
268 cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1]
269 if debug: main.log.info("Cut timestamp: " + cutTimestamp)
270
271 #validate link count and flow count
272 for i in range(0, 40):
273 main.ONOSbench.handle.sendline("onos $OC1 summary")
274 main.ONOSbench.handle.expect(":~")
275 linkCheck = main.ONOSbench.handle.before
276 #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
277 if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
278 break
279 if i == 39:
280 main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
281 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
282
283 time.sleep(5) #trying to avoid negative values
284
285 #intents events metrics installed timestamp
286 IEMtimestamps = [0]*(clusterCount + 1)
287 installedTemp = [0]*(clusterCount + 1)
288 for node in range(1, clusterCount +1):
289 cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
290 raw = ""
291 while "epoch)" not in raw:
292 main.ONOSbench.handle.sendline(cmd)
293 main.ONOSbench.handle.expect(":~")
294 raw = main.ONOSbench.handle.before
295
296 print(raw)
297
298 intentsTimestamps = {}
299 rawTimestamps = raw.splitlines()
300 for line in rawTimestamps:
301 if "Timestamp" in line and "grep" not in line:
302 metricKey = (line.split(" "))[1]
303 metricTimestamp = (line.split(" ")[len(line.split(" ")) -1]).replace("epoch)=","")
304 metricTimestamp = float(metricTimestamp)
305 metricTimestamp = numpy.divide(metricTimestamp, 1000)
306 if debug: main.log.info(repr(metricTimestamp))
307 intentsTimestamps[metricKey] = metricTimestamp
308 if metricKey == "Installed":
309 installedTemp[node] = metricTimestamp
310
311 main.log.info("Node: " + str(node) + " Timestamps: " + str(intentsTimestamps))
312 IEMtimestamps[node] = intentsTimestamps
313
314 myMax = max(installedTemp)
315 indexOfMax = installedTemp.index(myMax)
316
317 #number crunch
318 for metric in timestampMetrics: #this is where we sould add support for computing other timestamp metrics
319 if metric == "Installed":
320 if run >= warmUp:
321 main.log.report("link cut timestamp: " + cutTimestamp)
322 #readableInstalledTimestamp = str(intentsTimestamps["Installed"])
323 readableInstalledTimestamp = str(myMax)
324
325 #main.log.report("Intent Installed timestamp: " + str(intentsTimestamps["Installed"]))
326 main.log.report("Intent Installed timestamp: " + str(myMax))
327
328 cutEpoch = time.mktime(time.strptime(cutTimestamp, "%Y-%m-%d %H:%M:%S,%f"))
329 if debug: main.log.info("cutEpoch=" + str(cutEpoch))
330 #rerouteLatency = float(intentsTimestamps["Installed"] - cutEpoch)
331 rerouteLatency = float(myMax - cutEpoch)
332
333 rerouteLatency = numpy.divide(rerouteLatency, 1000)
334 main.log.report("Reroute latency:" + str(rerouteLatency) + " (seconds)\n ")
335 myResult[run-warmUp][0] = rerouteLatency
336 myResult[run-warmUp][1] = indexOfMax
337 if debug: main.log.info("Latency: " + str(myResult[run-warmUp][0]))
338 if debug: main.log.info("last node: " + str(myResult[run-warmUp][1]))
339
340 cmd = """ onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 up" """
341 if debug: main.log.info(cmd)
342 main.ONOSbench.handle.sendline(cmd)
343 main.ONOSbench.handle.expect(":~")
344
345
346
347 #wait for intent withdraw
348 main.ONOSbench.handle.sendline(withdrawCmd)
349 main.log.info(withdrawCmd)
350 main.ONOSbench.handle.expect(":~")
351 if debug: main.log.info(main.ONOSbench.handle.before)
352 main.ONOSbench.handle.sendline("onos $OC1 intents|grep WITHDRAWN|wc -l")
353 main.ONOSbench.handle.expect(":~")
354 intentWithdrawCheck = main.ONOSbench.handle.before
355 if (str(intents)) in intentWithdrawCheck:
356 main.log.info("intents withdrawn")
357 if debug: main.log.info(intentWithdrawCheck)
358
359 # wait for links to be reestablished
360 for i in range(0, 10):
361 main.ONOSbench.handle.sendline("onos $OC1 summary")
362 main.ONOSbench.handle.expect(":~")
363 linkCheck = main.ONOSbench.handle.before
364 if "links=16," in linkCheck:
365 break
366 time.sleep(1)
367 if i == 9:
368 main.log.info("Links Failed to reconnect, next iteration of data invalid." + linkCheck)
369
370 if run < warmUp:
371 main.log.info("Warm up run " + str(run+1) + " completed")
372
373 if debug: main.log.info(myResult)
374 latTemp = []
375 nodeTemp = []
376 for i in myResult:
377 latTemp.append(i[0])
378 nodeTemp.append(i[1])
379
380 mode = {}
381 for i in nodeTemp:
382 if i in mode:
383 mode[i] += 1
384 else:
385 mode[i] = 1
386
387 for i in mode.keys():
388 if mode[i] == max(mode.values()):
389 nodeMode = i
390
391 average = numpy.average(latTemp)
392 stdDev = numpy.std(latTemp)
393
394 average = numpy.multiply(average, 1000)
395 stdDev = numpy.multiply(stdDev, 1000)
396
397 main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents))
398 main.log.report("Latency average:................" + str(average))
399 main.log.report("Latency standard deviation:....." + str(stdDev))
400 main.log.report("Mode of last node to respond:..." + str(nodeMode))
401 main.log.report("________________________________________________________")
402
403 resultsDB = open("IntentRerouteLatDB", "a")
404 resultsDB.write("'" + commit + "',")
405 resultsDB.write(str(clusterCount) + ",")
406 resultsDB.write(str(intents) + ",")
407 resultsDB.write(str(average) + ",")
408 resultsDB.write(str(stdDev) + "\n")
409 resultsDB.close()
410
411 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
412