Gitiles
Code Review Sign In
gerrit.onosproject.org / OnosSystemTest / e4bc7131e5955a9b7008097bff1b2c90a069f0aa / . / TestON / tests / SCPFintentRerouteLatWithFlowObj / SCPFintentRerouteLatWithFlowObj.py
blob: a7ecb43eec3aed95bc9e6f9ece599d23e31b75d0 [file] [log] [blame]
YPZhang737d0012016-03-24 13:56:24 -0700[diff] [blame]1# ScaleOutTemplate
2#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
7import sys
8import os.path
9
10
11class SCPFintentRerouteLatWithFlowObj:
12
13 def __init__( self ):
14 self.default = ''
15
16 def CASE1( self, main ):
17
18 import time
19
20 global init
21 try:
22 if type(init) is not bool:
23 init = False
24 except NameError:
25 init = False
26
27 #Load values from params file
28 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
29 gitPull = main.params[ 'GIT' ][ 'autopull' ]
30 cellName = main.params[ 'ENV' ][ 'cellName' ]
31 Apps = main.params[ 'ENV' ][ 'cellApps' ]
32 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
33 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
34 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
35 main.maxNodes = int(main.params[ 'max' ])
36 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
37 cellName = main.params[ 'ENV' ][ 'cellName' ]
38
39 # -- INIT SECTION, ONLY RUNS ONCE -- #
40 if init == False:
41 init = True
42 global clusterCount #number of nodes running
43 global ONOSIp #list of ONOS IP addresses
44 global scale
45 global commit
46
47 clusterCount = 0
48 ONOSIp = [ 0 ]
49 scale = (main.params[ 'SCALE' ]).split(",")
50 clusterCount = int(scale[0])
51
52 #Populate ONOSIp with ips from params
53 ONOSIp = [0]
54 ONOSIp.extend(main.ONOSbench.getOnosIps())
55
56 print("-----------------" + str(ONOSIp))
57 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
58 if skipMvn != "yes":
59 mvnResult = main.ONOSbench.cleanInstall()
60
61 #git
62 main.step( "Git checkout and pull " + checkoutBranch )
63 if gitPull == 'on':
64 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
65 pullResult = main.ONOSbench.gitPull()
66
67 else:
68 checkoutResult = main.TRUE
69 pullResult = main.TRUE
70 main.log.info( "Skipped git checkout and pull" )
71
72 commit = main.ONOSbench.getVersion()
73 commit = (commit.split(" "))[1]
74
75 resultsDB = open("/tmp/IntentRerouteLatDBWithFlowObj", "w+")
76 resultsDB.close()
77
78 # -- END OF INIT SECTION --#
79
80 clusterCount = int(scale[0])
81 scale.remove(scale[0])
82
83 #kill off all onos processes
84 main.log.step("Safety check, killing all ONOS processes")
85 main.log.step("before initiating environment setup")
86 for node in range(1, main.maxNodes + 1):
87 main.ONOSbench.onosDie(ONOSIp[node])
88
89 #Uninstall everywhere
90 main.log.step( "Cleaning Enviornment..." )
91 for i in range(1, main.maxNodes + 1):
92 main.log.info(" Uninstalling ONOS " + str(i) )
93 main.ONOSbench.onosUninstall( ONOSIp[i] )
94
95 #construct the cell file
96 main.log.info("Creating cell file")
97 cellIp = []
98 for node in range (1, clusterCount + 1):
99 cellIp.append(ONOSIp[node])
100
101 main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), cellIp)
102
103 main.step( "Set Cell" )
104 main.ONOSbench.setCell(cellName)
105
106 main.step( "Creating ONOS package" )
107 packageResult = main.ONOSbench.onosPackage()
108
109 main.step( "verify cells" )
110 verifyCellResult = main.ONOSbench.verifyCell()
111
112 main.log.report( "Initializing " + str( clusterCount ) + " node cluster." )
113 for node in range(1, clusterCount + 1):
114 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
115 main.ONOSbench.onosInstall( ONOSIp[node])
116
117 for node in range(1, clusterCount + 1):
118 for i in range( 2 ):
119 isup = main.ONOSbench.isup( ONOSIp[node] )
120 if isup:
121 main.log.info("ONOS " + str(node) + " is up\n")
122 break
123 if not isup:
124 main.log.report( "ONOS " + str(node) + " didn't start!" )
125 main.log.info("Startup sequence complete")
126
127 deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
128 print("Device mastership list: " + str(deviceMastership))
129
130 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.store.flow.impl.NewDistributedFlowRuleStore", "backupEnabled false")
131
132 main.log.step("Setting up null provider")
133 for i in range(3):
134 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "deviceCount 8")
135 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "topoShape reroute")
136 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "enabled true")
137 main.ONOS1cli.setCfg( "org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator", "useFlowObjectives", "true")
138 time.sleep(5)
139 main.ONOSbench.handle.sendline("onos $OC1 summary")
140 main.ONOSbench.handle.expect(":~")
141 x = main.ONOSbench.handle.before
142 if "devices=8" in x and "links=16," in x:
143 break
144
145 index = 1
146 for node in deviceMastership:
147 for attempt in range(0,10):
148 cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)] + """ master" """)
149 main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
150 main.ONOSbench.handle.sendline(cmd)
151 main.ONOSbench.handle.expect(":~")
152 time.sleep(4)
153
154 cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index))
155 main.log.info(cmd)
156 main.ONOSbench.handle.sendline(cmd)
157 main.ONOSbench.handle.expect(":~")
158 check = main.ONOSbench.handle.before
159 main.log.info("CHECK:\n" + check)
160 if ("master=" + ONOSIp[int(node)]) in check:
161 break
162 index += 1
163
164 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
165
166 def CASE2( self, main ):
167
168 import time
169 import numpy
170 import datetime
171 #from scipy import stats
172
173 ts = time.time()
174
175 sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
176 warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
177 intentsList = (main.params[ 'TEST' ][ 'intents' ]).split(",")
178 debug = main.params[ 'TEST' ][ 'debug' ]
179 for i in range(0,len(intentsList)):
180 intentsList[i] = int(intentsList[i])
181
182 timestampMetrics = []
183 if main.params['METRICS']['Submitted'] == "1":
184 timestampMetrics.append("Submitted")
185 if main.params['METRICS']['Installed'] == "1":
186 timestampMetrics.append("Installed")
187 if main.params['METRICS']['Failed'] == "1":
188 timestampMetrics.append("Failed")
189 if main.params['METRICS']['Withdraw'] == "1":
190 timestampMetrics.append("Withdraw")
191 if main.params['METRICS']['Withdrawn'] == "1":
192 timestampMetrics.append("Withdrawn")
193 if debug: main.log.info(timestampMetrics)
194
195 if debug == "True":
196 debug = True
197 else:
198 debug = False
199
200 ingress = "null:0000000000000001"
201 egress = "null:0000000000000007"
202
203 for intents in intentsList:
204 main.log.report("Intent Batch size: " + str(intents) + "\n ")
205 myResult = [["latency", "lastNode"] for x in range(sampleSize)]
206
207 for run in range(0, (warmUp + sampleSize)):
208 if run > warmUp:
209 main.log.info("Starting test iteration " + str(run-warmUp))
210
211 cmd = """onos $OC1 "push-test-intents -i """
212 cmd += ingress + "/0 "
213 cmd += egress + "/0 "
214 cmd += str(intents) +""" 1" """
215 if debug: main.log.info(cmd)
216
217 withdrawCmd = cmd.replace("intents -i", "intents -w ")
218
219 #push-test-intents
220 main.ONOSbench.handle.sendline(cmd)
221 main.ONOSbench.handle.expect(":~")
222 myRawResult = main.ONOSbench.handle.before
223
224 for i in range(0, 40):
225 main.ONOSbench.handle.sendline("onos $OC1 summary")
226 main.ONOSbench.handle.expect(":~")
227 linkCheck = main.ONOSbench.handle.before
228 if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
229 break
230 if i == 39:
231 main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
232 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
233 #main.ONOSbench.logReport(ONOSIp[(clusterCount-1)], ["ERROR", "WARNING", "EXCEPT"], "d")
234 main.ONOSbench.sendline("onos $OC1 summary")
235 main.ONOSbench.sendline("onos $OC1 devices")
236 main.ONOSbench.sendline("onos $OC1 links")
237 main.ONOSbench.expect(":~")
238 main.log.info(main.ONOSbench.before)
239
240 #collect timestamp from link cut
241 cmd = """onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 down" """
242 if debug: main.log.info("COMMAND: " + str(cmd))
243 main.ONOSbench.handle.sendline(cmd)
244 main.ONOSbench.handle.expect(":~")
245
246 cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
247 for i in range(0,10):
248 main.ONOSbench.handle.sendline(cmd)
249 time.sleep(2)
250 main.ONOSbench.handle.expect(":~")
251 raw = main.ONOSbench.handle.before
252 #if "NullLinkProvider" in raw and "links=14" in raw:
253 if "links=14" in raw:
254 break
255 if i >= 9:
256 main.log.error("Expected output not being recieved... continuing")
257 main.log.info(raw)
258 break
259 time.sleep(2)
260
261 if debug: main.log.debug("raw: " + raw)
262
263 temp = raw.splitlines()
264
265 if debug: main.log.debug("temp (after splitlines): " + str(temp))
266
267 # Since the string is deterministic the date is always the 3rd element.
268 # However, if the data were grepping for in the onos log changes then this will
269 # not work. This is why we print out the raw and temp string so we can visually
270 # check if everything is in the correct order. temp should like this:
271 # temp = ['/onos$ onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep Top ',
272 # 'ologyManager| tail -1', '2015-10-15 12:03:33,736 ... ]
273 temp = temp[2]
274
275 if debug: main.log.debug("temp (checking for date): " + str(temp))
276
277 cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1]
278
279 if debug: main.log.info("Cut timestamp: " + cutTimestamp)
280
281 #validate link count and flow count
282 for i in range(0, 40):
283 main.ONOSbench.handle.sendline("onos $OC1 summary")
284 main.ONOSbench.handle.expect(":~")
285 linkCheck = main.ONOSbench.handle.before
286 #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
287 if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
288 break
289 if i == 39:
290 main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
291 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
292
293 time.sleep(5) #trying to avoid negative values
294
295 #intents events metrics installed timestamp
296 IEMtimestamps = [0]*(clusterCount + 1)
297 installedTemp = [0]*(clusterCount + 1)
298 for node in range(1, clusterCount +1):
299 cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
300 raw = ""
301 while "epoch)" not in raw:
302 main.ONOSbench.handle.sendline(cmd)
303 main.ONOSbench.handle.expect(":~")
304 raw = main.ONOSbench.handle.before
305
306 print(raw)
307
308 intentsTimestamps = {}
309 rawTimestamps = raw.splitlines()
310 for line in rawTimestamps:
311 if "Timestamp" in line and "grep" not in line:
312 metricKey = (line.split(" "))[1]
313 metricTimestamp = (line.split(" ")[len(line.split(" ")) -1]).replace("epoch)=","")
314 metricTimestamp = float(metricTimestamp)
315 metricTimestamp = numpy.divide(metricTimestamp, 1000)
316 if debug: main.log.info(repr(metricTimestamp))
317 intentsTimestamps[metricKey] = metricTimestamp
318 if metricKey == "Installed":
319 installedTemp[node] = metricTimestamp
320
321 main.log.info("Node: " + str(node) + " Timestamps: " + str(intentsTimestamps))
322 IEMtimestamps[node] = intentsTimestamps
323
324 myMax = max(installedTemp)
325 indexOfMax = installedTemp.index(myMax)
326
327 #number crunch
328 for metric in timestampMetrics: #this is where we sould add support for computing other timestamp metrics
329 if metric == "Installed":
330 if run >= warmUp:
331 main.log.report("link cut timestamp: " + cutTimestamp)
332 #readableInstalledTimestamp = str(intentsTimestamps["Installed"])
333 readableInstalledTimestamp = str(myMax)
334
335 #main.log.report("Intent Installed timestamp: " + str(intentsTimestamps["Installed"]))
336 main.log.report("Intent Installed timestamp: " + str(myMax))
337
338 cutEpoch = time.mktime(time.strptime(cutTimestamp, "%Y-%m-%d %H:%M:%S,%f"))
339 if debug: main.log.info("cutEpoch=" + str(cutEpoch))
340 #rerouteLatency = float(intentsTimestamps["Installed"] - cutEpoch)
341 rerouteLatency = float(myMax - cutEpoch)
342
343 rerouteLatency = numpy.divide(rerouteLatency, 1000)
344 main.log.report("Reroute latency:" + str(rerouteLatency) + " (seconds)\n ")
345 myResult[run-warmUp][0] = rerouteLatency
346 myResult[run-warmUp][1] = indexOfMax
347 if debug: main.log.info("Latency: " + str(myResult[run-warmUp][0]))
348 if debug: main.log.info("last node: " + str(myResult[run-warmUp][1]))
349
350 cmd = """ onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 up" """
351 if debug: main.log.info(cmd)
352 main.ONOSbench.handle.sendline(cmd)
353 main.ONOSbench.handle.expect(":~")
354
355 #wait for intent withdraw
356 main.ONOSbench.handle.sendline(withdrawCmd)
357 main.log.info(withdrawCmd)
358 main.ONOSbench.handle.expect(":~")
359 if debug: main.log.info(main.ONOSbench.handle.before)
360 main.ONOSbench.handle.sendline("onos $OC1 intents|grep WITHDRAWN|wc -l")
361 main.ONOSbench.handle.expect(":~")
362 intentWithdrawCheck = main.ONOSbench.handle.before
363 if (str(intents)) in intentWithdrawCheck:
364 main.log.info("intents withdrawn")
365 if debug: main.log.info(intentWithdrawCheck)
366
367 # wait for links to be reestablished
368 for i in range(0, 10):
369 main.ONOSbench.handle.sendline("onos $OC1 summary")
370 main.ONOSbench.handle.expect(":~")
371 linkCheck = main.ONOSbench.handle.before
372 if "links=16," in linkCheck:
373 break
374 time.sleep(1)
375 if i == 9:
376 main.log.info("Links Failed to reconnect, next iteration of data invalid." + linkCheck)
377
378 if run < warmUp:
379 main.log.info("Warm up run " + str(run+1) + " completed")
380
381 if debug: main.log.info(myResult)
382 latTemp = []
383 nodeTemp = []
384 for i in myResult:
385 latTemp.append(i[0])
386 nodeTemp.append(i[1])
387
388 mode = {}
389 for i in nodeTemp:
390 if i in mode:
391 mode[i] += 1
392 else:
393 mode[i] = 1
394
395 for i in mode.keys():
396 if mode[i] == max(mode.values()):
397 nodeMode = i
398
399 average = numpy.average(latTemp)
400 stdDev = numpy.std(latTemp)
401
402 average = numpy.multiply(average, 1000)
403 stdDev = numpy.multiply(stdDev, 1000)
404
405 main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents))
406 main.log.report("Latency average:................" + str(average))
407 main.log.report("Latency standard deviation:....." + str(stdDev))
408 main.log.report("Mode of last node to respond:..." + str(nodeMode))
409 main.log.report("________________________________________________________")
410
411 resultsDB = open("/tmp/IntentRerouteLatDBWithFlowObj", "a")
412 resultsDB.write("'" + commit + "',")
413 resultsDB.write(str(clusterCount) + ",")
414 resultsDB.write(str(intents) + ",")
415 resultsDB.write(str(average) + ",")
416 resultsDB.write(str(stdDev) + "\n")
417 resultsDB.close()
418
419 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])