blob: 182f4b6e8a07b17c4e86dd5b679434560a3ee9ad [file] [log] [blame]
YPZhangfebf7302016-05-24 16:45:56 -07001# SCPFintentRerouteLat
2"""
3SCPFintentRerouteLat
4 - Test Intent Reroute Latency
5 - Test Algorithm:
6 1. Start Null Provider reroute Topology
7 2. Using Push-test-intents to push batch size intents from switch 1 to switch 7
8 3. Cut the link between switch 3 and switch 4 (the path will reroute to switch 8)
9 4. Get the topology time stamp
10 5. Get Intent reroute(Installed) time stamp from each nodes
11 6. Use the latest intent time stamp subtract topology time stamp
12 - This test will run 5 warm up by default, warm up iteration can be setup in Param file
13 - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file
14 - The unit of the latency result is milliseconds
15"""
cameron@onlab.us78b89652015-07-08 15:21:03 -070016
17class SCPFintentRerouteLat:
YPZhangfebf7302016-05-24 16:45:56 -070018 def __init__(self):
cameron@onlab.us78b89652015-07-08 15:21:03 -070019 self.default = ''
20
YPZhangfebf7302016-05-24 16:45:56 -070021 def CASE0( self, main ):
22 '''
23 - GIT
24 - BUILDING ONOS
25 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
26 This step is usually skipped. Because in a Jenkins driven automated
27 test env. We want Jenkins jobs to pull&build for flexibility to handle
28 different versions of ONOS.
29 - Construct tests variables
30 '''
31 gitPull = main.params['GIT']['gitPull']
32 gitBranch = main.params['GIT']['gitBranch']
cameron@onlab.us78b89652015-07-08 15:21:03 -070033
YPZhangfebf7302016-05-24 16:45:56 -070034 main.case("Pull onos branch and build onos on Teststation.")
35
36 if gitPull == 'True':
37 main.step("Git Checkout ONOS branch: " + gitBranch)
38 stepResult = main.ONOSbench.gitCheckout(branch=gitBranch)
39 utilities.assert_equals(expect=main.TRUE,
40 actual=stepResult,
41 onpass="Successfully checkout onos branch.",
42 onfail="Failed to checkout onos branch. Exiting test...")
43 if not stepResult: main.exit()
44
45 main.step("Git Pull on ONOS branch:" + gitBranch)
46 stepResult = main.ONOSbench.gitPull()
47 utilities.assert_equals(expect=main.TRUE,
48 actual=stepResult,
49 onpass="Successfully pull onos. ",
50 onfail="Failed to pull onos. Exiting test ...")
51 if not stepResult: main.exit()
52
53 main.step("Building ONOS branch: " + gitBranch)
54 stepResult = main.ONOSbench.cleanInstall(skipTest=True)
55 utilities.assert_equals(expect=main.TRUE,
56 actual=stepResult,
57 onpass="Successfully build onos.",
58 onfail="Failed to build onos. Exiting test...")
59 if not stepResult: main.exit()
60
61 else:
62 main.log.warn("Skipped pulling onos and Skipped building ONOS")
63
64 main.apps = main.params['ENV']['cellApps']
65 main.BENCHUser = main.params['BENCH']['user']
66 main.BENCHIp = main.params['BENCH']['ip1']
67 main.MN1Ip = main.params['MN']['ip1']
68 main.maxNodes = int(main.params['max'])
69 main.skipMvn = main.params['TEST']['skipCleanInstall']
70 main.cellName = main.params['ENV']['cellName']
71 main.scale = (main.params['SCALE']).split(",")
72 main.dbFileName = main.params['DATABASE']['file']
73 main.timeout = int(main.params['SLEEP']['timeout'])
74 main.startUpSleep = int(main.params['SLEEP']['startup'])
75 main.installSleep = int(main.params['SLEEP']['install'])
76 main.verifySleep = int(main.params['SLEEP']['verify'])
77 main.verifyAttempts = int(main.params['ATTEMPTS']['verify'])
78 main.sampleSize = int(main.params['TEST']['sampleSize'])
79 main.warmUp = int(main.params['TEST']['warmUp'])
80 main.intentsList = (main.params['TEST']['intents']).split(",")
81 main.ingress = main.params['TEST']['ingress']
82 main.egress = main.params['TEST']['egress']
83 main.debug = main.params['TEST']['debug']
84 for i in range(0, len(main.intentsList)):
85 main.intentsList[i] = int(main.intentsList[i])
86 # Create DataBase file
87 main.log.info("Create Database file " + main.dbFileName)
88 resultsDB = open(main.dbFileName, "w+")
89 resultsDB.close()
90
91 def CASE1( self, main ):
92 '''
93 clean up test environment and set up
94 '''
cameron@onlab.us78b89652015-07-08 15:21:03 -070095 import time
96
YPZhangfebf7302016-05-24 16:45:56 -070097 main.log.info("Get ONOS cluster IP")
98 print(main.scale)
99 main.numCtrls = int(main.scale[0])
100 main.ONOSip = []
101 main.maxNumBatch = 0
102 main.AllONOSip = main.ONOSbench.getOnosIps()
103 for i in range(main.numCtrls):
104 main.ONOSip.append(main.AllONOSip[i])
105 main.log.info(main.ONOSip)
106 main.CLIs = []
107 main.log.info("Creating list of ONOS cli handles")
108 for i in range(main.numCtrls):
109 main.CLIs.append(getattr(main, 'ONOS%scli' % (i + 1)))
cameron@onlab.us78b89652015-07-08 15:21:03 -0700110
YPZhangfebf7302016-05-24 16:45:56 -0700111 if not main.CLIs:
112 main.log.error("Failed to create the list of ONOS cli handles")
113 main.cleanup()
114 main.exit()
cameron@onlab.us78b89652015-07-08 15:21:03 -0700115
YPZhangfebf7302016-05-24 16:45:56 -0700116 main.commit = main.ONOSbench.getVersion(report=True)
117 main.commit = main.commit.split(" ")[1]
118 main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls)
119 main.log.info("Safety check, killing all ONOS processes" +
120 " before initiating environment setup")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700121
YPZhangfebf7302016-05-24 16:45:56 -0700122 for i in range(main.numCtrls):
123 main.ONOSbench.onosDie(main.ONOSip[i])
cameron@onlab.us78b89652015-07-08 15:21:03 -0700124
YPZhangfebf7302016-05-24 16:45:56 -0700125 main.log.info("NODE COUNT = %s" % main.numCtrls)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700126
YPZhangfebf7302016-05-24 16:45:56 -0700127 # tempOnosIp = []
128 # for i in range(main.numCtrls):
129 # tempOnosIp.append(main.AllONOSip[i])
130 # print(tempOnosIp)
131 main.ONOSbench.createCellFile(main.ONOSbench.ip_address,
132 main.cellName,
133 main.MN1Ip,
134 main.apps,
135 main.ONOSip)
136 main.step("Apply cell to environment")
137 cellResult = main.ONOSbench.setCell(main.cellName)
138 verifyResult = main.ONOSbench.verifyCell()
139 stepResult = cellResult and verifyResult
140 utilities.assert_equals(expect=main.TRUE,
141 actual=stepResult,
142 onpass="Successfully applied cell to " + \
143 "environment",
144 onfail="Failed to apply cell to environment ")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700145
YPZhangfebf7302016-05-24 16:45:56 -0700146 main.step("Creating ONOS package")
Jon Hall4ba53f02015-07-29 13:07:41 -0700147 packageResult = main.ONOSbench.onosPackage()
YPZhangfebf7302016-05-24 16:45:56 -0700148 stepResult = packageResult
149 utilities.assert_equals(expect=main.TRUE,
150 actual=stepResult,
151 onpass="Successfully created ONOS package",
152 onfail="Failed to create ONOS package")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700153
YPZhangfebf7302016-05-24 16:45:56 -0700154 main.step("Uninstall ONOS package on all Nodes")
155 uninstallResult = main.TRUE
156 for i in range(int(main.numCtrls)):
157 main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i])
158 u_result = main.ONOSbench.onosUninstall(main.ONOSip[i])
159 utilities.assert_equals(expect=main.TRUE, actual=u_result,
160 onpass="Test step PASS",
161 onfail="Test step FAIL")
162 uninstallResult = (uninstallResult and u_result)
Jon Hall4ba53f02015-07-29 13:07:41 -0700163
YPZhangfebf7302016-05-24 16:45:56 -0700164 main.step("Install ONOS package on all Nodes")
165 installResult = main.TRUE
166 for i in range(int(main.numCtrls)):
167 main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i])
168 i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i])
169 utilities.assert_equals(expect=main.TRUE, actual=i_result,
170 onpass="Test step PASS",
171 onfail="Test step FAIL")
172 installResult = installResult and i_result
cameron@onlab.us78b89652015-07-08 15:21:03 -0700173
YPZhangfebf7302016-05-24 16:45:56 -0700174 main.step("Verify ONOS nodes UP status")
175 statusResult = main.TRUE
176 for i in range(int(main.numCtrls)):
177 main.log.info("ONOS Node " + main.ONOSip[i] + " status:")
178 onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
179 utilities.assert_equals(expect=main.TRUE, actual=onos_status,
180 onpass="Test step PASS",
181 onfail="Test step FAIL")
182 statusResult = (statusResult and onos_status)
183 time.sleep(2)
184 main.step("Start ONOS CLI on all nodes")
185 cliResult = main.TRUE
186 main.log.step(" Start ONOS cli using thread ")
187 startCliResult = main.TRUE
188 pool = []
189 main.threadID = 0
190 for i in range(int(main.numCtrls)):
191 t = main.Thread(target=main.CLIs[i].startOnosCli,
192 threadID=main.threadID,
193 name="startOnosCli",
194 args=[main.ONOSip[i]],
195 kwargs={"onosStartTimeout": main.timeout})
196 pool.append(t)
197 t.start()
198 main.threadID = main.threadID + 1
199 for t in pool:
200 t.join()
201 startCliResult = startCliResult and t.result
202 time.sleep(main.startUpSleep)
Jon Hall4ba53f02015-07-29 13:07:41 -0700203
YPZhangfebf7302016-05-24 16:45:56 -0700204 # configure apps
205 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=8)
206 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="reroute")
207 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
208 main.CLIs[0].setCfg("org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled", value="false")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700209
YPZhangfebf7302016-05-24 16:45:56 -0700210 time.sleep(main.startUpSleep)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700211
YPZhangfebf7302016-05-24 16:45:56 -0700212 # Balance Master
213 main.CLIs[0].balanceMasters()
214 if len(main.ONOSip) > 1:
215 main.CLIs[0].deviceRole("null:0000000000000003", main.ONOSip[0])
216 main.CLIs[0].deviceRole("null:0000000000000004", main.ONOSip[0])
cameron@onlab.us78b89652015-07-08 15:21:03 -0700217
218 def CASE2( self, main ):
cameron@onlab.us78b89652015-07-08 15:21:03 -0700219 import time
220 import numpy
221 import datetime
YPZhangfebf7302016-05-24 16:45:56 -0700222 import json
223 # from scipy import stats
cameron@onlab.us78b89652015-07-08 15:21:03 -0700224
225 ts = time.time()
YPZhangfebf7302016-05-24 16:45:56 -0700226 print(main.intentsList)
227 for batchSize in main.intentsList:
228 main.log.report("Intent Batch size: " + str(batchSize) + "\n ")
229 main.LatencyList = []
230 for run in range(0, (main.warmUp + main.sampleSize)):
231 if run >= main.warmUp:
232 main.log.info("================================================")
233 main.log.info("Starting test iteration " + str(run - main.warmUp))
234 main.log.info("================================================")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700235 else:
YPZhangfebf7302016-05-24 16:45:56 -0700236 main.log.info("====================Warm Up=====================")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700237
YPZhangfebf7302016-05-24 16:45:56 -0700238 # push intents
239 main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
240 offset=1, options="-i", timeout=main.timeout)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700241
YPZhangfebf7302016-05-24 16:45:56 -0700242 # check links and flows
243 k = 0
244 verify = main.FALSE
245 linkCheck = 0
246 flowsCheck = 0
247 while k <= main.verifyAttempts:
248 time.sleep(main.verifySleep)
249 summary = json.loads(main.CLIs[0].summary(timeout=main.timeout))
250 linkCheck = summary.get("links")
251 flowsCheck = summary.get("flows")
252 if linkCheck == 16 and flowsCheck == batchSize * 7:
253 main.log.info("links: {}, flows: {} ".format(linkCheck, flowsCheck))
254 verify = main.TRUE
255 break
256 k += 1
257 if not verify:
258 main.log.warn("Links or flows number are not match!")
259 main.log.warn("links: {}, flows: {} ".format(linkCheck, flowsCheck))
260 continue
cameron@onlab.us78b89652015-07-08 15:21:03 -0700261
YPZhangfebf7302016-05-24 16:45:56 -0700262 # Bring link down
263 main.CLIs[0].link("0000000000000004/1", "0000000000000003/2", "down",
264 timeout=main.timeout, showResponse=False)
265 verify = main.FALSE
266 k = 0
267 topoManagerLog = ""
268 while k <= main.verifyAttempts:
269 time.sleep(main.verifySleep)
270 summary = json.loads(main.CLIs[0].summary(timeout=main.timeout))
271 linkCheck = summary.get("links")
272 flowsCheck = summary.get("flows")
273 if linkCheck == 14:
274 main.log.info("links: {}, flows: {} ".format(linkCheck, flowsCheck))
275 verify = main.TRUE
276 break
277 k += 1
278 if not verify:
279 main.log.warn("Links number are not match in TopologyManager log!")
280 main.log.warn(topoManagerLog)
281 continue
cameron@onlab.us78b89652015-07-08 15:21:03 -0700282
YPZhangfebf7302016-05-24 16:45:56 -0700283 try:
284 # expect twice to clean the pexpect buffer
285 main.ONOSbench.handle.sendline("")
286 main.ONOSbench.handle.expect("\$")
287 main.ONOSbench.handle.expect("\$")
288 # send line by using bench, can't use driver because pexpect buffer problem
289 cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
290 main.ONOSbench.handle.sendline(cmd)
291 time.sleep(1)
292 main.ONOSbench.handle.expect(":~")
293 topoManagerLog = main.ONOSbench.handle.before
294 topoManagerLogTemp = topoManagerLog.splitlines()
295 # To make sure we get correct topology log
296 for lines in topoManagerLogTemp:
297 if "creationTime" in lines:
298 topoManagerLog = lines
299 main.log.info("Topology Manager log:")
300 print(topoManagerLog)
301 cutTimestamp = float(topoManagerLog.split("creationTime=")[1].split(",")[0])
302 except:
303 main.log.error("Topology Log is not correct!")
304 print(topoManagerLog)
305 # If we got wrong Topology log, we should skip this iteration, and continue for next one
306 continue
307
308 installedTemp = []
309 time.sleep(1)
310 for cli in main.CLIs:
311 tempJson = json.loads(cli.intentsEventsMetrics())
312 Installedtime = tempJson.get('intentInstalledTimestamp').get('value')
313 installedTemp.append(float(Installedtime))
314 for i in range(0, len(installedTemp)):
315 main.log.info("ONOS Node {} Installed Time stemp: {}".format((i + 1), installedTemp[i]))
316 maxInstallTime = float(max(installedTemp))
317 if run >= main.warmUp:
318 main.log.info("Installed time stemp: {0:f}".format(maxInstallTime))
319 main.log.info("CutTimestamp: {0:f}".format(cutTimestamp))
320 # Both timeStemps are milliseconds
321 main.log.info("Latency: {0:f}".format(float(maxInstallTime - cutTimestamp)))
322 main.LatencyList.append(float(maxInstallTime - cutTimestamp))
323
324 # Verify Summary after we bring up link, and withdrawn intents
325 main.CLIs[0].link("0000000000000004/1", "0000000000000003/2", "up",
326 timeout=main.timeout)
327 k = 0
328 verify = main.FALSE
329 linkCheck = 0
330 flowsCheck = 0
331 while k <= main.verifyAttempts:
332 time.sleep(main.verifySleep)
333 main.CLIs[0].removeAllIntents(purge=True, sync=True, timeout=main.timeout)
334 time.sleep(1)
335 main.CLIs[0].purgeWithdrawnIntents()
336 summary = json.loads(main.CLIs[0].summary())
337 linkCheck = summary.get("links")
338 flowsCheck = summary.get("flows")
339 intentCheck = summary.get("intents")
340 if linkCheck == 16 and flowsCheck == 0 and intentCheck == 0:
341 main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
342 verify = main.TRUE
343 break
344 k += 1
345 if not verify:
346 main.log.error("links, flows, or intents are not correct!")
347 main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
348 continue
349
350 aveLatency = 0
351 stdLatency = 0
352 aveLatency = numpy.average(main.LatencyList)
353 stdLatency = numpy.std(main.LatencyList)
354 main.log.report("Scale: " + str(main.numCtrls) + " \tIntent batch: " + str(batchSize))
355 main.log.report("Latency average:................" + str(aveLatency))
356 main.log.report("Latency standard deviation:....." + str(stdLatency))
cameron@onlab.us78b89652015-07-08 15:21:03 -0700357 main.log.report("________________________________________________________")
358
YPZhangfebf7302016-05-24 16:45:56 -0700359 resultsDB = open(main.dbFileName, "a")
360 resultsDB.write("'" + main.commit + "',")
361 resultsDB.write(str(main.numCtrls) + ",")
362 resultsDB.write(str(batchSize) + ",")
363 resultsDB.write(str(aveLatency) + ",")
364 resultsDB.write(str(stdLatency) + "\n")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700365 resultsDB.close()
YPZhangfebf7302016-05-24 16:45:56 -0700366 del main.scale[0]