Gitiles
Code Review Sign In
gerrit.onosproject.org / OnosSystemTest / 1201bc45f1482bee6076b0a50d50834fc5d9b288 / . / TestON / tests / IntentRerouteLat / IntentRerouteLat.py
blob: 7198108fb682d896343257cc6cf5668701c9a55c [file] [log] [blame]
cameron@onlab.us1201bc42015-04-01 16:30:05 -0700[diff] [blame^]1# ScaleOutTemplate - IntentReroute
2#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
7import sys
8import os.path
9
10
11class IntentRerouteLat:
12
13 def __init__( self ):
14 self.default = ''
15
16 def CASE1( self, main ): #This is the initialization case
17 #this case will clean up all nodes
18 import time #but only node 1 is started in this case
19 import os.path
20
21 global clusterCount #number of nodes running
22 global ONOSIp #list of ONOS IP addresses
23 clusterCount = 1
24 ONOSIp = [ 0 ]
25
26
27 #Load values from params file
28 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
29 gitPull = main.params[ 'GIT' ][ 'autopull' ]
30 cellName = main.params[ 'ENV' ][ 'cellName' ]
31 Features= main.params[ 'ENV' ][ 'cellFeatures' ]
32 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
33 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
34 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
35 maxNodes = int(main.params[ 'availableNodes' ])
36 Features = main.params[ 'ENV' ][ 'cellFeatures' ]
37 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
38 switchCount = (main.params[ 'ENV' ][ 'scale1switches' ]).split(",")
39 homeDir = os.path.expanduser('~')
40
41 #Populate ONOSIp with ips from params
42 for i in range(1, maxNodes + 1):
43 ipString = 'ip' + str(i)
44 ONOSIp.append(main.params[ 'CTRL' ][ ipString ])
45
46 tempIp = []
47 for node in range( 1, clusterCount + 1):
48 tempIp.append(ONOSIp[node])
49
50 #kill off all onos processes
51 main.log.step("Safety check, killing all ONOS processes")
52 main.log.step("before initiating enviornment setup")
53 for node in range(1, maxNodes + 1):
54 main.ONOSbench.onosDie(ONOSIp[node])
55
56 #construct the cell file
57 main.log.info("Creating cell file")
58 exec "a = main.ONOSbench.createCellFile"
59 cellIp = []
60 for node in range (1, clusterCount + 1):
61 cellIp.append(ONOSIp[node])
62 a(BENCHIp,cellName,MN1Ip,str(Features), *cellIp)
63
64 main.step( "Applying cell file to environment" )
65 cellApplyResult = main.ONOSbench.setCell( cellName )
66
67 #Uninstall everywhere
68 main.log.step( "Cleaning Enviornment..." )
69 for i in range(1, maxNodes + 1):
70 main.log.info(" Uninstalling ONOS " + str(i) )
71 main.ONOSbench.onosUninstall( ONOSIp[i] )
72
73 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
74 if skipMvn != "yes":
75 mvnResult = main.ONOSbench.cleanInstall()
76
77 #git
78 main.step( "Git checkout and pull " + checkoutBranch )
79 if gitPull == 'on':
80 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
81 pullResult = main.ONOSbench.gitPull()
82
83 else:
84 checkoutResult = main.TRUE
85 pullResult = main.TRUE
86 main.log.info( "Skipped git checkout and pull" )
87
88 main.ONOSbench.handle.sendline("cp ~/ONLabTest/TestON/dependencies/IntentRerouteTest/oneNode7linear.cfg ~/onos/tools/package/etc/")
89 main.ONOSbench.handle.expect(":~")
90 main.ONOSbench.handle.sendline("cp ~/ONLabTest/TestON/dependencies/IntentRerouteTest/oneNode7linearCut.cfg ~/onos/tools/package/etc/")
91 main.ONOSbench.handle.expect(":~")
92
93 main.ONOSbench.createNullDevProviderFile(BENCHIp, tempIp, switchCount)
94 main.ONOSbench.createNullLinkProviderFile(BENCHIp, fileName=("/opt/onos/apache-karaf-3.0.2/etc/oneNode7linear.cfg"))
95
96 main.step( "Creating ONOS package" )
97 packageResult = main.ONOSbench.onosPackage()
98
99 main.step( "Installing ONOS package" )
100 install1Result = main.ONOSbench.onosInstall( node=ONOSIp[1] )
101
102 main.step( "verify cells" )
103 verifyCellResult = main.ONOSbench.verifyCell()
104
105 main.step( "Set cell for ONOS cli env" )
106 cli1 = main.ONOS1cli.startOnosCli( ONOSIp[1] )
107
108
109 def CASE2( self, main ):
110 # This case increases the cluster size by whatever scale is
111 # Note: 'scale' is the size of the step
112 # if scaling is not a part of your test, simply run this case
113 # once after CASE1 to set up your enviornment for your desired
114 # cluster size. If scaling is a part of you test call this case each time
115 # you want to increase cluster size
116
117 ''
118 'Increase number of nodes and initiate CLI'
119 ''
120 import time
121 global clusterCount
122
123 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
124 scale = int( main.params[ 'SCALE' ] )
125 clusterCount += scale
126 homeDir = os.path.expanduser('~')
127 switchParams = ("scale" + str(clusterCount) + "switches")
128 switchCount = (main.params[ 'ENV' ][ switchParams ]).split(",")
129
130 if clusterCount == 1:
131 temp = "one"
132 if clusterCount == 3:
133 temp = "three"
134 if clusterCount == 5:
135 temp = "five"
136 if clusterCount == 7:
137 temp = "seven"
138
139 lgfile = temp + "Node7linear.cfg"
140 main.ONOSbench.handle.sendline("cp ~/ONLabTest/TestON/dependencies/IntentRerouteTest/" + lgfile + " ~/onos/tools/package/etc/")
141 main.ONOSbench.handle.expect(":~")
142 main.ONOSbench.handle.sendline("cp ~/ONLabTest/TestON/dependencies/IntentRerouteTest/" + temp + "Node7linearCut.cfg" + " ~/onos/tools/package/etc/")
143 main.ONOSbench.handle.expect(":~")
144
145 main.log.info("Creating cell file")
146 exec "a = main.ONOSbench.createCellFile"
147 cellIp = []
148 for node in range (1, clusterCount + 1):
149 cellIp.append(ONOSIp[node])
150 a(BENCHIp,cellName,MN1Ip,str(Features), *cellIp)
151
152 main.step( "Applying cell file to environment" )
153 cellApplyResult = main.ONOSbench.setCell( cellName )
154
155 #Uninstall everywhere
156 main.log.step( "Cleaning Enviornment..." )
157 for node in range(1, maxNodes + 1):
158 main.ONOSbench.onosDie(ONOSIp[node])
159 main.log.info(" Uninstalling ONOS " + str(node) )
160 main.ONOSbench.onosUninstall( ONOSIp[node] )
161
162 tempIp = []
163 for node in range( 1, clusterCount + 1):
164 tempIp.append(ONOSIp[node])
165
166 main.ONOSbench.createNullDevProviderFile(BENCHIp, tempIp, switchCount)
167 main.ONOSbench.createNullLinkProviderFile(BENCHIp, fileName=("/opt/onos/apache-karaf-3.0.2/etc/" + lgfile))
168
169 main.ONOSbench.onosPackage()
170
171
172 main.log.report( "Increasing cluster size to " + str( clusterCount ) )
173 for node in range(1, clusterCount + 1):
174 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
175 main.ONOSbench.onosInstall( node=ONOSIp[node])
176 if node == 1:
177 main.ONOS1cli.startOnosCli( ONOSIp[1] )
178
179 for node in range(1, clusterCount + 1):
180 for i in range( 2 ):
181 isup = main.ONOSbench.isup( ONOSIp[node] )
182 if isup:
183 main.log.info("ONOS " + str(node) + " is up\n")
184 break
185 if not isup:
186 main.log.report( "ONOS " + str(node) + " didn't start!" )
187
188 def CASE3( self, main ):
189
190 import time
191 import numpy
192 import datetime
193 #from scipy import stats
194
195 ts = time.time()
196 date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d')
197
198 sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
199 warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
200 intentsList = (main.params[ 'TEST' ][ 'intents' ]).split(",")
201 debug = main.params[ 'TEST' ][ 'debug' ]
202 for i in range(0,len(intentsList)):
203 intentsList[i] = int(intentsList[i])
204
205 timestampMetrics = []
206 if main.params['METRICS']['Submitted'] == "1":
207 timestampMetrics.append("Submitted")
208 if main.params['METRICS']['Installed'] == "1":
209 timestampMetrics.append("Installed")
210 if main.params['METRICS']['Failed'] == "1":
211 timestampMetrics.append("Failed")
212 if main.params['METRICS']['Withdraw'] == "1":
213 timestampMetrics.append("Withdraw")
214 if main.params['METRICS']['Withdrawn'] == "1":
215 timestampMetrics.append("Withdrawn")
216 if debug: main.log.info(timestampMetrics)
217
218 if debug == "True":
219 debug = True
220 else:
221 debug = False
222
223 if clusterCount == 1:
224 temp = "one"
225 if clusterCount == 3:
226 temp = "three"
227 if clusterCount == 5:
228 temp = "five"
229 if clusterCount == 7:
230 temp = "seven"
231
232 lgfile = temp + "Node7linear.cfg"
233 lgfileCut = temp + "Node7linearCut.cfg"
234 linkCount = 0
235
236 for i in range(0,15):
237 main.ONOSbench.handle.sendline("onos $OC1 links|wc -l")
238 main.ONOSbench.handle.expect(":~")
239 linkCount = main.ONOSbench.handle.before
240 if debug: main.log.info("Link Count check: " + linkCount)
241 if str(16) in linkCount:
242 break
243 time.sleep(3)
244 main.log.info("waiting for links")
245
246 links = "--"
247 while "=null:" not in links:
248 if debug: main.log.info("top of loop")
249 main.ONOSbench.handle.sendline("onos $OC1 links")
250 main.ONOSbench.handle.expect(":~")
251 links = main.ONOSbench.handle.before
252 if debug: main.log.info(str(links))
253 time.sleep(1)
254 links = links.splitlines()
255 templinks = links
256
257 tempDevices = []
258 for line in links:
259 temp = line.split(" ")
260 temp[0].replace("src=","")
261 temp[0] = (temp[0].split("/"))[0]
262 tempDevices.append(temp[0])
263
264 tempDevices.sort()
265 devices = []
266 for i in tempDevices:
267 if "src=null" in i:
268 devices.append(i.replace("src=", ""))
269 if debug: main.log.info(str(devices))
270
271 ingress = devices[0]
272 egress = devices.pop()
273 if debug: main.log.info("ingress: " + ingress)
274 if debug: main.log.info("egress: " + egress)
275
276
277 for intents in intentsList:
278 main.log.report("Intent Batch size: " + str(intents) + "\n ")
279 myResult = [["latency", "lastNode"] for x in range(sampleSize)]
280
281 for run in range(0, (warmUp + sampleSize)):
282 if run > warmUp:
283 main.log.info("Starting test iteration " + str(run-warmUp))
284
285 cmd = """onos $OC1 "push-test-intents -i """
286 cmd += ingress + "/0 "
287 cmd += egress + "/0 "
288 cmd += str(intents) +""" 1" """
289 if debug: main.log.info(cmd)
290
291 withdrawCmd = cmd.replace("intents -i", "intents -w ")
292
293 #push-test-intents
294 main.ONOSbench.handle.sendline(cmd)
295 main.ONOSbench.handle.expect(":~")
296 myRawResult = main.ONOSbench.handle.before
297
298 for i in range(0, 40):
299 main.ONOSbench.handle.sendline("onos $OC1 summary")
300 main.ONOSbench.handle.expect(":~")
301 linkCheck = main.ONOSbench.handle.before
302 if ("flows=" + str(8*intents) + ",") in linkCheck:
303 break
304 if i == 39:
305 main.log.error("Flow count incorrect, data invalid.")
306
307 #cut link
308 nodesLinksCut = []
309 if clusterCount == 1:
310 nodesLinksCut.append(1)
311 if clusterCount == 3 or clusterCount == 5:
312 nodesLinksCut.append(2)
313 if clusterCount == 7:
314 nodesLinksCut.append(3)
315 nodesLinksCut.append(4)
316
317 for node in nodesLinksCut:
318 exec "a = main.ONOS%s.createNullLinkProviderFile" %str(node)
319 a(ONOSIp[node], fileName=("/opt/onos/apache-karaf-3.0.2/etc/" + lgfileCut), onNode=True)
320
321 #collect timestamp from link cut
322 cmd = "onos-ssh $OC" + str(nodesLinksCut[len(nodesLinksCut)-1]) + " cat /opt/onos/log/karaf.log | grep " + lgfileCut + "| tail -1"
323 if debug: main.log.info("COMMAND: " + str(cmd))
324
325 for i in range(0,10):
326 main.ONOSbench.handle.sendline(cmd)
327 main.ONOSbench.handle.expect(":~")
328 raw = main.ONOSbench.handle.before
329 if "NullLinkProvider" in raw:
330 break
331 if i >= 9:
332 main.log.error("Expected output not being recieved... continuing")
333 main.log.info(raw)
334 break
335 time.sleep(2)
336
337 temp = raw.splitlines()
338 for line in temp:
339 if str(date) in line:
340 temp = line
341 break
342
343 cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1]
344
345 #validate link count and flow count
346 for i in range(0, 40):
347 main.ONOSbench.handle.sendline("onos $OC1 summary")
348 main.ONOSbench.handle.expect(":~")
349 linkCheck = main.ONOSbench.handle.before
350 if "links=14," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
351 break
352 if i == 39:
353 main.log.error("Link or flow count incorrect, data invalid.")
354
355 #intents events metrics installed timestamp
356 IEMtimestamps = [0]*(clusterCount + 1)
357 installedTemp = [0]*(clusterCount + 1)
358 for node in range(1, clusterCount +1):
359 cmd = "onos $OC" + str(node) + " intents-events-metrics|grep Timestamp"
360 raw = ""
361 while "Timestamp" not in raw:
362 main.ONOSbench.handle.sendline(cmd)
363 main.ONOSbench.handle.expect(":~")
364 raw = main.ONOSbench.handle.before
365
366 intentsTimestamps = {}
367 rawTimestamps = raw.splitlines()
368 for line in rawTimestamps:
369 if "Timestamp" in line and "grep" not in line:
370 metricKey = (line.split(" "))[1]
371 metricTimestamp = (line.split(" ")[len(line.split(" ")) -1]).replace("epoch)=","")
372 metricTimestamp = float(metricTimestamp)
373 metricTimestamp = numpy.divide(metricTimestamp, 1000)
374 if debug: main.log.info(repr(metricTimestamp))
375 intentsTimestamps[metricKey] = metricTimestamp
376 if metricKey == "Installed":
377 installedTemp[node] = metricTimestamp
378
379 main.log.info("Node: " + str(node) + " Timestamps: " + str(intentsTimestamps))
380 IEMtimestamps[node] = intentsTimestamps
381
382 myMax = max(installedTemp)
383 indexOfMax = installedTemp.index(myMax)
384
385 #number crunch
386 for metric in timestampMetrics: #this is where we sould add support for computing other timestamp metrics
387 if metric == "Installed":
388 if run >= warmUp:
389 main.log.report("link cut timestamp: " + cutTimestamp)
390 #readableInstalledTimestamp = str(intentsTimestamps["Installed"])
391 readableInstalledTimestamp = str(myMax)
392
393 #main.log.report("Intent Installed timestamp: " + str(intentsTimestamps["Installed"]))
394 main.log.report("Intent Installed timestamp: " + str(myMax))
395
396 cutEpoch = time.mktime(time.strptime(cutTimestamp, "%Y-%m-%d %H:%M:%S,%f"))
397 if debug: main.log.info("cutEpoch=" + str(cutEpoch))
398 #rerouteLatency = float(intentsTimestamps["Installed"] - cutEpoch)
399 rerouteLatency = float(myMax - cutEpoch)
400
401 rerouteLatency = numpy.divide(rerouteLatency, 1000)
402 main.log.report("Reroute latency:" + str(rerouteLatency) + " (seconds)\n ")
403 myResult[run-warmUp][0] = rerouteLatency
404 myResult[run-warmUp][1] = indexOfMax
405 if debug: main.log.info("Latency: " + str(myResult[run-warmUp][0]))
406 if debug: main.log.info("last node: " + str(myResult[run-warmUp][1]))
407
408 #time.sleep(12)
409
410 #reset
411 for node in nodesLinksCut:
412 exec "a = main.ONOS%s.createNullLinkProviderFile" %str(node)
413 a(ONOSIp[node], fileName=("/opt/onos/apache-karaf-3.0.2/etc/" + lgfile), onNode=True)
414
415 #wait for intent withdraw
416 if debug: main.log.info(withdrawCmd)
417 main.ONOSbench.handle.sendline(withdrawCmd)
418 main.ONOSbench.handle.expect(":~")
419 main.ONOSbench.handle.sendline("onos $OC1 intents|grep WITHDRAWN|wc -l")
420 main.ONOSbench.handle.expect(":~")
421 intentWithdrawCheck = main.ONOSbench.handle.before
422 if (str(intents)) in intentWithdrawCheck:
423 main.log.info("intents withdrawn")
424 if debug: main.log.info(intentWithdrawCheck)
425
426 # wait for links to be reestablished
427 for i in range(0, 10):
428 main.ONOSbench.handle.sendline("onos $OC1 summary")
429 main.ONOSbench.handle.expect(":~")
430 linkCheck = main.ONOSbench.handle.before
431 if "links=16," in linkCheck:
432 break
433 time.sleep(1)
434 if i == 9:
435 main.log.info("Links Failed to reconnect, next iteration of data invalid.")
436
437 if run < warmUp:
438 main.log.info("Warm up run " + str(run+1) + " completed")
439
440 if debug: main.log.info(myResult)
441 latTemp = []
442 nodeTemp = []
443 for i in myResult:
444 latTemp.append(i[0])
445 nodeTemp.append(i[1])
446
447 mode = {}
448 for i in nodeTemp:
449 if i in mode:
450 mode[i] += 1
451 else:
452 mode[i] = 1
453
454 for i in mode.keys():
455 if mode[i] == max(mode.values()):
456 nodeMode = i
457
458 average = numpy.average(latTemp)
459 stdDev = numpy.std(latTemp)
460
461 main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents))
462 main.log.report("Latency average:................" + str(average))
463 main.log.report("Latency standard deviation:....." + str(stdDev))
464 main.log.report("Mode of last node to respond:..." + str(nodeMode))
465 main.log.report("________________________________________________________")
466
467
468