Gitiles
Code Review Sign In
gerrit.onosproject.org / OnosSystemTest / 763ee04955cd794f14d9cb4ba5fd5bb002b35f0b / . / TestON / tests / IntentRerouteLat / IntentRerouteLat.py
blob: f824a5cf6b3ea58b73edee5208b53912b888ab34 [file] [log] [blame]
pingping-lin763ee042015-05-20 17:45:30 -0700[diff] [blame^]1# ScaleOutTemplate
2#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
7import sys
8import os.path
9
10
11class IntentRerouteLat:
12
13 def __init__( self ):
14 self.default = ''
15
16 def CASE1( self, main ):
17
18 import time
19 global init
20 try:
21 if type(init) is not bool:
22 init = False
23 except NameError:
24 init = False
25
26 #Load values from params file
27 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
28 gitPull = main.params[ 'GIT' ][ 'autopull' ]
29 cellName = main.params[ 'ENV' ][ 'cellName' ]
30 Apps = main.params[ 'ENV' ][ 'cellApps' ]
31 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
32 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
33 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
34 maxNodes = int(main.params[ 'availableNodes' ])
35 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
36 cellName = main.params[ 'ENV' ][ 'cellName' ]
37
38 # -- INIT SECTION, ONLY RUNS ONCE -- #
39 if init == False:
40 init = True
41 global clusterCount #number of nodes running
42 global ONOSIp #list of ONOS IP addresses
43 global scale
44 global commit
45
46 clusterCount = 0
47 ONOSIp = [ 0 ]
48 scale = (main.params[ 'SCALE' ]).split(",")
49 clusterCount = int(scale[0])
50
51 #Populate ONOSIp with ips from params
52 for i in range(1, maxNodes + 1):
53 ipString = 'ip' + str(i)
54 ONOSIp.append(main.params[ 'CTRL' ][ ipString ])
55
56 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
57 if skipMvn != "yes":
58 mvnResult = main.ONOSbench.cleanInstall()
59
60 #git
61 main.step( "Git checkout and pull " + checkoutBranch )
62 if gitPull == 'on':
63 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
64 pullResult = main.ONOSbench.gitPull()
65
66 else:
67 checkoutResult = main.TRUE
68 pullResult = main.TRUE
69 main.log.info( "Skipped git checkout and pull" )
70
71 commit = main.ONOSbench.getVersion()
72 commit = (commit.split(" "))[1]
73
74 resultsDB = open("IntentRerouteLatDB", "w+")
75 resultsDB.close()
76
77 # -- END OF INIT SECTION --#
78
79 clusterCount = int(scale[0])
80 scale.remove(scale[0])
81
82 #kill off all onos processes
83 main.log.step("Safety check, killing all ONOS processes")
84 main.log.step("before initiating enviornment setup")
85 for node in range(1, maxNodes + 1):
86 main.ONOSbench.onosDie(ONOSIp[node])
87
88 #Uninstall everywhere
89 main.log.step( "Cleaning Enviornment..." )
90 for i in range(1, maxNodes + 1):
91 main.log.info(" Uninstalling ONOS " + str(i) )
92 main.ONOSbench.onosUninstall( ONOSIp[i] )
93
94 #construct the cell file
95 main.log.info("Creating cell file")
96 cellIp = []
97 for node in range (1, clusterCount + 1):
98 cellIp.append(ONOSIp[node])
99
100 main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), *cellIp)
101
102 main.step( "Set Cell" )
103 main.ONOSbench.setCell(cellName)
104
105 main.step( "Creating ONOS package" )
106 packageResult = main.ONOSbench.onosPackage()
107
108 main.step( "verify cells" )
109 verifyCellResult = main.ONOSbench.verifyCell()
110
111 main.log.report( "Initializing " + str( clusterCount ) + " node cluster." )
112 for node in range(1, clusterCount + 1):
113 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
114 main.ONOSbench.onosInstall( ONOSIp[node])
115
116 for node in range(1, clusterCount + 1):
117 for i in range( 2 ):
118 isup = main.ONOSbench.isup( ONOSIp[node] )
119 if isup:
120 main.log.info("ONOS " + str(node) + " is up\n")
121 break
122 if not isup:
123 main.log.report( "ONOS " + str(node) + " didn't start!" )
124 main.log.info("Startup sequence complete")
125
126 deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
127 print("Device mastership list: " + str(deviceMastership))
128
129 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount 8 " """)
130 main.ONOSbench.handle.expect(":~")
131 print repr(main.ONOSbench.handle.before)
132 time.sleep(3)
133 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape reroute" """)
134 main.ONOSbench.handle.expect(":~")
135 print repr(main.ONOSbench.handle.before)
136 time.sleep(3)
137
138 time.sleep(10)
139
140 once = True
141 for attempt in range(0,10):
142 main.ONOSbench.handle.sendline("onos $OC1 summary")
143 main.ONOSbench.handle.expect(":~")
144 x = main.ONOSbench.handle.before
145 main.log.info("Initial setup. ONOS summary response: \n" + x + "\n")
146
147 if "devices=8" in x and "links=16," in x:
148 break
149 else:
150 if once == False:
151 main.log.error("Null provider start failed, retrying..")
152 once = False
153 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled false" """)
154 main.ONOSbench.handle.expect(":~")
155 time.sleep(3)
156 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled true" """)
157 main.ONOSbench.handle.expect(":~")
158 time.sleep(8)
159
160 index = 1
161 for node in deviceMastership:
162 for attempt in range(0,10):
163 cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)] + """ master" """)
164 main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
165 main.ONOSbench.handle.sendline(cmd)
166 main.ONOSbench.handle.expect(":~")
167 time.sleep(4)
168
169 cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index))
170 main.log.info(cmd)
171 main.ONOSbench.handle.sendline(cmd)
172 main.ONOSbench.handle.expect(":~")
173 check = main.ONOSbench.handle.before
174 main.log.info("CHECK:\n" + check)
175 if ("master=" + ONOSIp[int(node)]) in check:
176 break
177 index += 1
178
179
180 def CASE2( self, main ):
181
182 import time
183 import numpy
184 import datetime
185 #from scipy import stats
186
187 ts = time.time()
188 date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d')
189
190 sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
191 warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
192 intentsList = (main.params[ 'TEST' ][ 'intents' ]).split(",")
193 debug = main.params[ 'TEST' ][ 'debug' ]
194 for i in range(0,len(intentsList)):
195 intentsList[i] = int(intentsList[i])
196
197 timestampMetrics = []
198 if main.params['METRICS']['Submitted'] == "1":
199 timestampMetrics.append("Submitted")
200 if main.params['METRICS']['Installed'] == "1":
201 timestampMetrics.append("Installed")
202 if main.params['METRICS']['Failed'] == "1":
203 timestampMetrics.append("Failed")
204 if main.params['METRICS']['Withdraw'] == "1":
205 timestampMetrics.append("Withdraw")
206 if main.params['METRICS']['Withdrawn'] == "1":
207 timestampMetrics.append("Withdrawn")
208 if debug: main.log.info(timestampMetrics)
209
210 if debug == "True":
211 debug = True
212 else:
213 debug = False
214
215 ingress = "null:0000000000000001"
216 egress = "null:0000000000000007"
217
218 for intents in intentsList:
219 main.log.report("Intent Batch size: " + str(intents) + "\n ")
220 myResult = [["latency", "lastNode"] for x in range(sampleSize)]
221
222 for run in range(0, (warmUp + sampleSize)):
223 if run > warmUp:
224 main.log.info("Starting test iteration " + str(run-warmUp))
225
226 cmd = """onos $OC1 "push-test-intents -i """
227 cmd += ingress + "/0 "
228 cmd += egress + "/0 "
229 cmd += str(intents) +""" 1" """
230 if debug: main.log.info(cmd)
231
232 withdrawCmd = cmd.replace("intents -i", "intents -w ")
233
234 #push-test-intents
235 main.ONOSbench.handle.sendline(cmd)
236 main.ONOSbench.handle.expect(":~")
237 myRawResult = main.ONOSbench.handle.before
238
239 for i in range(0, 40):
240 main.ONOSbench.handle.sendline("onos $OC1 summary")
241 main.ONOSbench.handle.expect(":~")
242 linkCheck = main.ONOSbench.handle.before
243 if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
244 break
245 if i == 39:
246 main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
247
248
249 #collect timestamp from link cut
250 cmd = """onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 down" """
251 if debug: main.log.info("COMMAND: " + str(cmd))
252 main.ONOSbench.handle.sendline(cmd)
253
254 cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
255 for i in range(0,10):
256 main.ONOSbench.handle.sendline(cmd)
257 time.sleep(2)
258 main.ONOSbench.handle.expect(":~")
259 raw = main.ONOSbench.handle.before
260 #if "NullLinkProvider" in raw and "links=14" in raw:
261 if "links=14" in raw:
262 break
263 if i >= 9:
264 main.log.error("Expected output not being recieved... continuing")
265 main.log.info(raw)
266 break
267 time.sleep(2)
268
269 temp = raw.splitlines()
270 for line in temp:
271 if str(date) in line:
272 temp = line
273 break
274
275 cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1]
276 if debug: main.log.info("Cut timestamp: " + cutTimestamp)
277
278 #validate link count and flow count
279 for i in range(0, 40):
280 main.ONOSbench.handle.sendline("onos $OC1 summary")
281 main.ONOSbench.handle.expect(":~")
282 linkCheck = main.ONOSbench.handle.before
283 #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
284 if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
285 break
286 if i == 39:
287 main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
288
289 time.sleep(5) #trying to avoid negative values
290
291 #intents events metrics installed timestamp
292 IEMtimestamps = [0]*(clusterCount + 1)
293 installedTemp = [0]*(clusterCount + 1)
294 for node in range(1, clusterCount +1):
295 cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
296 raw = ""
297 while "epoch)" not in raw:
298 main.ONOSbench.handle.sendline(cmd)
299 main.ONOSbench.handle.expect(":~")
300 raw = main.ONOSbench.handle.before
301
302 print(raw)
303
304 intentsTimestamps = {}
305 rawTimestamps = raw.splitlines()
306 for line in rawTimestamps:
307 if "Timestamp" in line and "grep" not in line:
308 metricKey = (line.split(" "))[1]
309 metricTimestamp = (line.split(" ")[len(line.split(" ")) -1]).replace("epoch)=","")
310 metricTimestamp = float(metricTimestamp)
311 metricTimestamp = numpy.divide(metricTimestamp, 1000)
312 if debug: main.log.info(repr(metricTimestamp))
313 intentsTimestamps[metricKey] = metricTimestamp
314 if metricKey == "Installed":
315 installedTemp[node] = metricTimestamp
316
317 main.log.info("Node: " + str(node) + " Timestamps: " + str(intentsTimestamps))
318 IEMtimestamps[node] = intentsTimestamps
319
320 myMax = max(installedTemp)
321 indexOfMax = installedTemp.index(myMax)
322
323 #number crunch
324 for metric in timestampMetrics: #this is where we sould add support for computing other timestamp metrics
325 if metric == "Installed":
326 if run >= warmUp:
327 main.log.report("link cut timestamp: " + cutTimestamp)
328 #readableInstalledTimestamp = str(intentsTimestamps["Installed"])
329 readableInstalledTimestamp = str(myMax)
330
331 #main.log.report("Intent Installed timestamp: " + str(intentsTimestamps["Installed"]))
332 main.log.report("Intent Installed timestamp: " + str(myMax))
333
334 cutEpoch = time.mktime(time.strptime(cutTimestamp, "%Y-%m-%d %H:%M:%S,%f"))
335 if debug: main.log.info("cutEpoch=" + str(cutEpoch))
336 #rerouteLatency = float(intentsTimestamps["Installed"] - cutEpoch)
337 rerouteLatency = float(myMax - cutEpoch)
338
339 rerouteLatency = numpy.divide(rerouteLatency, 1000)
340 main.log.report("Reroute latency:" + str(rerouteLatency) + " (seconds)\n ")
341 myResult[run-warmUp][0] = rerouteLatency
342 myResult[run-warmUp][1] = indexOfMax
343 if debug: main.log.info("Latency: " + str(myResult[run-warmUp][0]))
344 if debug: main.log.info("last node: " + str(myResult[run-warmUp][1]))
345
346 cmd = """ onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 up" """
347
348 #wait for intent withdraw
349 if debug: main.log.info(cmd)
350 main.ONOSbench.handle.sendline(cmd)
351 main.ONOSbench.handle.expect(":~")
352 main.ONOSbench.handle.sendline("onos $OC1 intents|grep WITHDRAWN|wc -l")
353 main.ONOSbench.handle.expect(":~")
354 intentWithdrawCheck = main.ONOSbench.handle.before
355 if (str(intents)) in intentWithdrawCheck:
356 main.log.info("intents withdrawn")
357 if debug: main.log.info(intentWithdrawCheck)
358
359 # wait for links to be reestablished
360 for i in range(0, 10):
361 main.ONOSbench.handle.sendline("onos $OC1 summary")
362 main.ONOSbench.handle.expect(":~")
363 linkCheck = main.ONOSbench.handle.before
364 if "links=16," in linkCheck:
365 break
366 time.sleep(1)
367 if i == 9:
368 main.log.info("Links Failed to reconnect, next iteration of data invalid." + linkCheck)
369
370 if run < warmUp:
371 main.log.info("Warm up run " + str(run+1) + " completed")
372
373 if debug: main.log.info(myResult)
374 latTemp = []
375 nodeTemp = []
376 for i in myResult:
377 latTemp.append(i[0])
378 nodeTemp.append(i[1])
379
380 mode = {}
381 for i in nodeTemp:
382 if i in mode:
383 mode[i] += 1
384 else:
385 mode[i] = 1
386
387 for i in mode.keys():
388 if mode[i] == max(mode.values()):
389 nodeMode = i
390
391 average = numpy.average(latTemp)
392 stdDev = numpy.std(latTemp)
393
394 average = numpy.multiply(average, 1000)
395 stdDev = numpy.multiply(stdDev, 1000)
396
397 main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents))
398 main.log.report("Latency average:................" + str(average))
399 main.log.report("Latency standard deviation:....." + str(stdDev))
400 main.log.report("Mode of last node to respond:..." + str(nodeMode))
401 main.log.report("________________________________________________________")
402
403 resultsDB = open("IntentRerouteLatDB", "a")
404 resultsDB.write("'" + commit + "',")
405 resultsDB.write(str(clusterCount) + ",")
406 resultsDB.write(str(intents) + ",")
407 resultsDB.write(str(average) + ",")
408 resultsDB.write(str(stdDev) + "\n")
409 resultsDB.close()
410