blob: 4a63f1807e9fcdf6252bff19b48bce8459e02368 [file] [log] [blame]
YPZhangfebf7302016-05-24 16:45:56 -07001# SCPFintentRerouteLat
2"""
3SCPFintentRerouteLat
4 - Test Intent Reroute Latency
5 - Test Algorithm:
6 1. Start Null Provider reroute Topology
7 2. Using Push-test-intents to push batch size intents from switch 1 to switch 7
8 3. Cut the link between switch 3 and switch 4 (the path will reroute to switch 8)
9 4. Get the topology time stamp
10 5. Get Intent reroute(Installed) time stamp from each nodes
11 6. Use the latest intent time stamp subtract topology time stamp
12 - This test will run 5 warm up by default, warm up iteration can be setup in Param file
13 - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file
14 - The unit of the latency result is milliseconds
15"""
cameron@onlab.us78b89652015-07-08 15:21:03 -070016
17class SCPFintentRerouteLat:
YPZhangfebf7302016-05-24 16:45:56 -070018 def __init__(self):
cameron@onlab.us78b89652015-07-08 15:21:03 -070019 self.default = ''
20
YPZhangfebf7302016-05-24 16:45:56 -070021 def CASE0( self, main ):
22 '''
23 - GIT
24 - BUILDING ONOS
25 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
26 This step is usually skipped. Because in a Jenkins driven automated
27 test env. We want Jenkins jobs to pull&build for flexibility to handle
28 different versions of ONOS.
29 - Construct tests variables
30 '''
31 gitPull = main.params['GIT']['gitPull']
32 gitBranch = main.params['GIT']['gitBranch']
cameron@onlab.us78b89652015-07-08 15:21:03 -070033
YPZhangfebf7302016-05-24 16:45:56 -070034 main.case("Pull onos branch and build onos on Teststation.")
35
36 if gitPull == 'True':
37 main.step("Git Checkout ONOS branch: " + gitBranch)
38 stepResult = main.ONOSbench.gitCheckout(branch=gitBranch)
39 utilities.assert_equals(expect=main.TRUE,
40 actual=stepResult,
41 onpass="Successfully checkout onos branch.",
42 onfail="Failed to checkout onos branch. Exiting test...")
43 if not stepResult: main.exit()
44
45 main.step("Git Pull on ONOS branch:" + gitBranch)
46 stepResult = main.ONOSbench.gitPull()
47 utilities.assert_equals(expect=main.TRUE,
48 actual=stepResult,
49 onpass="Successfully pull onos. ",
50 onfail="Failed to pull onos. Exiting test ...")
51 if not stepResult: main.exit()
52
53 main.step("Building ONOS branch: " + gitBranch)
54 stepResult = main.ONOSbench.cleanInstall(skipTest=True)
55 utilities.assert_equals(expect=main.TRUE,
56 actual=stepResult,
57 onpass="Successfully build onos.",
58 onfail="Failed to build onos. Exiting test...")
59 if not stepResult: main.exit()
60
61 else:
62 main.log.warn("Skipped pulling onos and Skipped building ONOS")
63
64 main.apps = main.params['ENV']['cellApps']
65 main.BENCHUser = main.params['BENCH']['user']
66 main.BENCHIp = main.params['BENCH']['ip1']
67 main.MN1Ip = main.params['MN']['ip1']
68 main.maxNodes = int(main.params['max'])
69 main.skipMvn = main.params['TEST']['skipCleanInstall']
70 main.cellName = main.params['ENV']['cellName']
71 main.scale = (main.params['SCALE']).split(",")
72 main.dbFileName = main.params['DATABASE']['file']
73 main.timeout = int(main.params['SLEEP']['timeout'])
74 main.startUpSleep = int(main.params['SLEEP']['startup'])
75 main.installSleep = int(main.params['SLEEP']['install'])
76 main.verifySleep = int(main.params['SLEEP']['verify'])
YPZhang2b9b26d2016-06-20 16:18:29 -070077 main.setMasterSleep = int(main.params['SLEEP']['setmaster'])
YPZhangfebf7302016-05-24 16:45:56 -070078 main.verifyAttempts = int(main.params['ATTEMPTS']['verify'])
79 main.sampleSize = int(main.params['TEST']['sampleSize'])
80 main.warmUp = int(main.params['TEST']['warmUp'])
81 main.intentsList = (main.params['TEST']['intents']).split(",")
82 main.ingress = main.params['TEST']['ingress']
83 main.egress = main.params['TEST']['egress']
84 main.debug = main.params['TEST']['debug']
85 for i in range(0, len(main.intentsList)):
86 main.intentsList[i] = int(main.intentsList[i])
87 # Create DataBase file
88 main.log.info("Create Database file " + main.dbFileName)
89 resultsDB = open(main.dbFileName, "w+")
90 resultsDB.close()
91
92 def CASE1( self, main ):
93 '''
94 clean up test environment and set up
95 '''
cameron@onlab.us78b89652015-07-08 15:21:03 -070096 import time
97
YPZhangfebf7302016-05-24 16:45:56 -070098 main.log.info("Get ONOS cluster IP")
99 print(main.scale)
100 main.numCtrls = int(main.scale[0])
101 main.ONOSip = []
102 main.maxNumBatch = 0
103 main.AllONOSip = main.ONOSbench.getOnosIps()
104 for i in range(main.numCtrls):
105 main.ONOSip.append(main.AllONOSip[i])
106 main.log.info(main.ONOSip)
107 main.CLIs = []
108 main.log.info("Creating list of ONOS cli handles")
109 for i in range(main.numCtrls):
110 main.CLIs.append(getattr(main, 'ONOS%scli' % (i + 1)))
cameron@onlab.us78b89652015-07-08 15:21:03 -0700111
YPZhangfebf7302016-05-24 16:45:56 -0700112 if not main.CLIs:
113 main.log.error("Failed to create the list of ONOS cli handles")
114 main.cleanup()
115 main.exit()
cameron@onlab.us78b89652015-07-08 15:21:03 -0700116
YPZhangfebf7302016-05-24 16:45:56 -0700117 main.commit = main.ONOSbench.getVersion(report=True)
118 main.commit = main.commit.split(" ")[1]
119 main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls)
120 main.log.info("Safety check, killing all ONOS processes" +
121 " before initiating environment setup")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700122
YPZhangfebf7302016-05-24 16:45:56 -0700123 for i in range(main.numCtrls):
124 main.ONOSbench.onosDie(main.ONOSip[i])
cameron@onlab.us78b89652015-07-08 15:21:03 -0700125
YPZhangfebf7302016-05-24 16:45:56 -0700126 main.log.info("NODE COUNT = %s" % main.numCtrls)
YPZhangfebf7302016-05-24 16:45:56 -0700127 main.ONOSbench.createCellFile(main.ONOSbench.ip_address,
128 main.cellName,
129 main.MN1Ip,
130 main.apps,
131 main.ONOSip)
132 main.step("Apply cell to environment")
133 cellResult = main.ONOSbench.setCell(main.cellName)
134 verifyResult = main.ONOSbench.verifyCell()
135 stepResult = cellResult and verifyResult
136 utilities.assert_equals(expect=main.TRUE,
137 actual=stepResult,
138 onpass="Successfully applied cell to " + \
139 "environment",
140 onfail="Failed to apply cell to environment ")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700141
YPZhangfebf7302016-05-24 16:45:56 -0700142 main.step("Creating ONOS package")
Jon Hall4ba53f02015-07-29 13:07:41 -0700143 packageResult = main.ONOSbench.onosPackage()
YPZhangfebf7302016-05-24 16:45:56 -0700144 stepResult = packageResult
145 utilities.assert_equals(expect=main.TRUE,
146 actual=stepResult,
147 onpass="Successfully created ONOS package",
148 onfail="Failed to create ONOS package")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700149
YPZhangfebf7302016-05-24 16:45:56 -0700150 main.step("Uninstall ONOS package on all Nodes")
151 uninstallResult = main.TRUE
152 for i in range(int(main.numCtrls)):
153 main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i])
154 u_result = main.ONOSbench.onosUninstall(main.ONOSip[i])
155 utilities.assert_equals(expect=main.TRUE, actual=u_result,
156 onpass="Test step PASS",
157 onfail="Test step FAIL")
158 uninstallResult = (uninstallResult and u_result)
Jon Hall4ba53f02015-07-29 13:07:41 -0700159
YPZhangfebf7302016-05-24 16:45:56 -0700160 main.step("Install ONOS package on all Nodes")
161 installResult = main.TRUE
162 for i in range(int(main.numCtrls)):
163 main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i])
164 i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i])
165 utilities.assert_equals(expect=main.TRUE, actual=i_result,
166 onpass="Test step PASS",
167 onfail="Test step FAIL")
168 installResult = installResult and i_result
cameron@onlab.us78b89652015-07-08 15:21:03 -0700169
YPZhangfebf7302016-05-24 16:45:56 -0700170 main.step("Verify ONOS nodes UP status")
171 statusResult = main.TRUE
172 for i in range(int(main.numCtrls)):
173 main.log.info("ONOS Node " + main.ONOSip[i] + " status:")
174 onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
175 utilities.assert_equals(expect=main.TRUE, actual=onos_status,
176 onpass="Test step PASS",
177 onfail="Test step FAIL")
178 statusResult = (statusResult and onos_status)
179 time.sleep(2)
180 main.step("Start ONOS CLI on all nodes")
181 cliResult = main.TRUE
Jon Hall6509dbf2016-06-21 17:01:17 -0700182 main.step(" Start ONOS cli using thread ")
YPZhangfebf7302016-05-24 16:45:56 -0700183 startCliResult = main.TRUE
184 pool = []
185 main.threadID = 0
186 for i in range(int(main.numCtrls)):
187 t = main.Thread(target=main.CLIs[i].startOnosCli,
188 threadID=main.threadID,
189 name="startOnosCli",
190 args=[main.ONOSip[i]],
191 kwargs={"onosStartTimeout": main.timeout})
192 pool.append(t)
193 t.start()
194 main.threadID = main.threadID + 1
195 for t in pool:
196 t.join()
197 startCliResult = startCliResult and t.result
198 time.sleep(main.startUpSleep)
Jon Hall4ba53f02015-07-29 13:07:41 -0700199
YPZhangfebf7302016-05-24 16:45:56 -0700200 # configure apps
201 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=8)
202 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="reroute")
203 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
204 main.CLIs[0].setCfg("org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled", value="false")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700205
YPZhangfebf7302016-05-24 16:45:56 -0700206 time.sleep(main.startUpSleep)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700207
YPZhangfebf7302016-05-24 16:45:56 -0700208 # Balance Master
209 main.CLIs[0].balanceMasters()
210 if len(main.ONOSip) > 1:
211 main.CLIs[0].deviceRole("null:0000000000000003", main.ONOSip[0])
212 main.CLIs[0].deviceRole("null:0000000000000004", main.ONOSip[0])
YPZhang2b9b26d2016-06-20 16:18:29 -0700213 time.sleep( main.setMasterSleep )
cameron@onlab.us78b89652015-07-08 15:21:03 -0700214
215 def CASE2( self, main ):
cameron@onlab.us78b89652015-07-08 15:21:03 -0700216 import time
217 import numpy
218 import datetime
YPZhangfebf7302016-05-24 16:45:56 -0700219 import json
220 # from scipy import stats
cameron@onlab.us78b89652015-07-08 15:21:03 -0700221
222 ts = time.time()
YPZhangfebf7302016-05-24 16:45:56 -0700223 print(main.intentsList)
224 for batchSize in main.intentsList:
225 main.log.report("Intent Batch size: " + str(batchSize) + "\n ")
226 main.LatencyList = []
YPZhang8742d2e2016-06-16 15:31:58 -0700227 validRun = 0
228 invalidRun = 0
229 while validRun <= main.warmUp + main.sampleSize and invalidRun <= 20:
230 if validRun >= main.warmUp:
YPZhangfebf7302016-05-24 16:45:56 -0700231 main.log.info("================================================")
YPZhang8742d2e2016-06-16 15:31:58 -0700232 main.log.info("Starting test iteration: {} ".format(validRun - main.warmUp))
233 main.log.info("Total iteration: {}".format(validRun + invalidRun))
YPZhangfebf7302016-05-24 16:45:56 -0700234 main.log.info("================================================")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700235 else:
YPZhangfebf7302016-05-24 16:45:56 -0700236 main.log.info("====================Warm Up=====================")
cameron@onlab.us78b89652015-07-08 15:21:03 -0700237
YPZhangfebf7302016-05-24 16:45:56 -0700238 # push intents
239 main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
240 offset=1, options="-i", timeout=main.timeout)
cameron@onlab.us78b89652015-07-08 15:21:03 -0700241
YPZhangfebf7302016-05-24 16:45:56 -0700242 # check links and flows
243 k = 0
244 verify = main.FALSE
245 linkCheck = 0
246 flowsCheck = 0
247 while k <= main.verifyAttempts:
248 time.sleep(main.verifySleep)
249 summary = json.loads(main.CLIs[0].summary(timeout=main.timeout))
250 linkCheck = summary.get("links")
251 flowsCheck = summary.get("flows")
252 if linkCheck == 16 and flowsCheck == batchSize * 7:
253 main.log.info("links: {}, flows: {} ".format(linkCheck, flowsCheck))
254 verify = main.TRUE
255 break
256 k += 1
257 if not verify:
258 main.log.warn("Links or flows number are not match!")
259 main.log.warn("links: {}, flows: {} ".format(linkCheck, flowsCheck))
YPZhang8742d2e2016-06-16 15:31:58 -0700260 # bring back topology
261 main.CLIs[0].removeAllIntents(purge=True, sync=True, timeout=main.timeout)
262 time.sleep(1)
263 main.CLIs[0].purgeWithdrawnIntents()
264 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=0)
265 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="false")
266 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=8)
267 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
268 if validRun >= main.warmUp:
269 invalidRun += 1
270 continue
271 else:
272 validRun += 1
273 continue
cameron@onlab.us78b89652015-07-08 15:21:03 -0700274
YPZhangfebf7302016-05-24 16:45:56 -0700275 # Bring link down
276 main.CLIs[0].link("0000000000000004/1", "0000000000000003/2", "down",
277 timeout=main.timeout, showResponse=False)
278 verify = main.FALSE
279 k = 0
280 topoManagerLog = ""
281 while k <= main.verifyAttempts:
282 time.sleep(main.verifySleep)
283 summary = json.loads(main.CLIs[0].summary(timeout=main.timeout))
284 linkCheck = summary.get("links")
285 flowsCheck = summary.get("flows")
286 if linkCheck == 14:
287 main.log.info("links: {}, flows: {} ".format(linkCheck, flowsCheck))
288 verify = main.TRUE
289 break
290 k += 1
291 if not verify:
292 main.log.warn("Links number are not match in TopologyManager log!")
293 main.log.warn(topoManagerLog)
YPZhang8742d2e2016-06-16 15:31:58 -0700294 # bring back topology
295 main.CLIs[0].removeAllIntents(purge=True, sync=True, timeout=main.timeout)
296 time.sleep(1)
297 main.CLIs[0].purgeWithdrawnIntents()
298 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=0)
299 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="false")
300 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=8)
301 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
302 if validRun >= main.warmUp:
303 invalidRun += 1
304 continue
305 else:
306 validRun += 1
307 continue
cameron@onlab.us78b89652015-07-08 15:21:03 -0700308
YPZhangfebf7302016-05-24 16:45:56 -0700309 try:
310 # expect twice to clean the pexpect buffer
311 main.ONOSbench.handle.sendline("")
312 main.ONOSbench.handle.expect("\$")
313 main.ONOSbench.handle.expect("\$")
314 # send line by using bench, can't use driver because pexpect buffer problem
315 cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
316 main.ONOSbench.handle.sendline(cmd)
317 time.sleep(1)
318 main.ONOSbench.handle.expect(":~")
319 topoManagerLog = main.ONOSbench.handle.before
320 topoManagerLogTemp = topoManagerLog.splitlines()
321 # To make sure we get correct topology log
322 for lines in topoManagerLogTemp:
323 if "creationTime" in lines:
324 topoManagerLog = lines
325 main.log.info("Topology Manager log:")
326 print(topoManagerLog)
327 cutTimestamp = float(topoManagerLog.split("creationTime=")[1].split(",")[0])
328 except:
329 main.log.error("Topology Log is not correct!")
330 print(topoManagerLog)
YPZhang8742d2e2016-06-16 15:31:58 -0700331 # bring back topology
332 verify = main.FALSE
333 main.CLIs[0].removeAllIntents(purge=True, sync=True, timeout=main.timeout)
334 time.sleep(1)
335 main.CLIs[0].purgeWithdrawnIntents()
336 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=0)
337 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="false")
338 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=8)
339 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
340 if validRun >= main.warmUp:
341 invalidRun += 1
342 else:
343 validRun += 1
YPZhangfebf7302016-05-24 16:45:56 -0700344 # If we got wrong Topology log, we should skip this iteration, and continue for next one
345 continue
346
347 installedTemp = []
348 time.sleep(1)
349 for cli in main.CLIs:
350 tempJson = json.loads(cli.intentsEventsMetrics())
351 Installedtime = tempJson.get('intentInstalledTimestamp').get('value')
352 installedTemp.append(float(Installedtime))
353 for i in range(0, len(installedTemp)):
354 main.log.info("ONOS Node {} Installed Time stemp: {}".format((i + 1), installedTemp[i]))
355 maxInstallTime = float(max(installedTemp))
YPZhang8742d2e2016-06-16 15:31:58 -0700356 if validRun >= main.warmUp and verify:
YPZhangfebf7302016-05-24 16:45:56 -0700357 main.log.info("Installed time stemp: {0:f}".format(maxInstallTime))
358 main.log.info("CutTimestamp: {0:f}".format(cutTimestamp))
359 # Both timeStemps are milliseconds
360 main.log.info("Latency: {0:f}".format(float(maxInstallTime - cutTimestamp)))
361 main.LatencyList.append(float(maxInstallTime - cutTimestamp))
YPZhang8742d2e2016-06-16 15:31:58 -0700362 # We get valid latency, validRun + 1
363 validRun += 1
YPZhangfebf7302016-05-24 16:45:56 -0700364
365 # Verify Summary after we bring up link, and withdrawn intents
366 main.CLIs[0].link("0000000000000004/1", "0000000000000003/2", "up",
367 timeout=main.timeout)
368 k = 0
369 verify = main.FALSE
370 linkCheck = 0
371 flowsCheck = 0
372 while k <= main.verifyAttempts:
373 time.sleep(main.verifySleep)
374 main.CLIs[0].removeAllIntents(purge=True, sync=True, timeout=main.timeout)
375 time.sleep(1)
376 main.CLIs[0].purgeWithdrawnIntents()
377 summary = json.loads(main.CLIs[0].summary())
378 linkCheck = summary.get("links")
379 flowsCheck = summary.get("flows")
380 intentCheck = summary.get("intents")
381 if linkCheck == 16 and flowsCheck == 0 and intentCheck == 0:
382 main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
383 verify = main.TRUE
384 break
385 k += 1
386 if not verify:
387 main.log.error("links, flows, or intents are not correct!")
388 main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
YPZhang8742d2e2016-06-16 15:31:58 -0700389 # bring back topology
390 main.log.info("Bring back topology...")
391 main.CLIs[0].removeAllIntents(purge=True, sync=True, timeout=main.timeout)
392 time.sleep(1)
393 main.CLIs[0].purgeWithdrawnIntents()
394 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=0)
395 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="false")
396 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=8)
397 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
YPZhangfebf7302016-05-24 16:45:56 -0700398 continue
399
400 aveLatency = 0
401 stdLatency = 0
402 aveLatency = numpy.average(main.LatencyList)
403 stdLatency = numpy.std(main.LatencyList)
404 main.log.report("Scale: " + str(main.numCtrls) + " \tIntent batch: " + str(batchSize))
405 main.log.report("Latency average:................" + str(aveLatency))
406 main.log.report("Latency standard deviation:....." + str(stdLatency))
cameron@onlab.us78b89652015-07-08 15:21:03 -0700407 main.log.report("________________________________________________________")
408
YPZhang8742d2e2016-06-16 15:31:58 -0700409 if not (numpy.isnan(aveLatency) or numpy.isnan(stdLatency)):
410 # check if got NaN for result
411 resultsDB = open(main.dbFileName, "a")
412 resultsDB.write("'" + main.commit + "',")
413 resultsDB.write(str(main.numCtrls) + ",")
414 resultsDB.write(str(batchSize) + ",")
415 resultsDB.write(str(aveLatency) + ",")
416 resultsDB.write(str(stdLatency) + "\n")
417 resultsDB.close()
YPZhangfebf7302016-05-24 16:45:56 -0700418 del main.scale[0]