blob: ce9155da8a3550340528772cef37b71197bd5ae3 [file] [log] [blame]
YPZhangfebf7302016-05-24 16:45:56 -07001# SCPFintentRerouteLatWithFlowObj
2"""
3SCPFintentRerouteLat
4 - Test Intent Reroute Latency
5 - Test Algorithm:
6 1. Start Null Provider reroute Topology
7 2. Using Push-test-intents to push batch size intents from switch 1 to switch 7
8 3. Cut the link between switch 3 and switch 4 (the path will reroute to switch 8)
9 4. Get the topology time stamp
10 5. Get Intent reroute(Installed) time stamp from each nodes
11 6. Use the latest intent time stamp subtract topology time stamp
12 - This test will run 5 warm up by default, warm up iteration can be setup in Param file
13 - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file
14 - The unit of the latency result is milliseconds
15 - Ues flowObject set to True
16"""
YPZhang737d0012016-03-24 13:56:24 -070017
18class SCPFintentRerouteLatWithFlowObj:
YPZhangfebf7302016-05-24 16:45:56 -070019 def __init__(self):
20 self.default=''
YPZhang737d0012016-03-24 13:56:24 -070021
YPZhangfebf7302016-05-24 16:45:56 -070022 def CASE0( self, main ):
23 '''
24 - GIT
25 - BUILDING ONOS
26 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
27 This step is usually skipped. Because in a Jenkins driven automated
28 test env. We want Jenkins jobs to pull&build for flexibility to handle
29 different versions of ONOS.
30 - Construct tests variables
31 '''
32 gitPull = main.params['GIT']['gitPull']
33 gitBranch = main.params['GIT']['gitBranch']
34
35 main.case("Pull onos branch and build onos on Teststation.")
36
37 if gitPull == 'True':
38 main.step("Git Checkout ONOS branch: " + gitBranch)
39 stepResult = main.ONOSbench.gitCheckout(branch=gitBranch)
40 utilities.assert_equals(expect=main.TRUE,
41 actual=stepResult,
42 onpass="Successfully checkout onos branch.",
43 onfail="Failed to checkout onos branch. Exiting test...")
44 if not stepResult: main.exit()
45
46 main.step("Git Pull on ONOS branch:" + gitBranch)
47 stepResult = main.ONOSbench.gitPull()
48 utilities.assert_equals(expect=main.TRUE,
49 actual=stepResult,
50 onpass="Successfully pull onos. ",
51 onfail="Failed to pull onos. Exiting test ...")
52 if not stepResult: main.exit()
53
54 main.step("Building ONOS branch: " + gitBranch)
55 stepResult = main.ONOSbench.cleanInstall(skipTest=True)
56 utilities.assert_equals(expect=main.TRUE,
57 actual=stepResult,
58 onpass="Successfully build onos.",
59 onfail="Failed to build onos. Exiting test...")
60 if not stepResult: main.exit()
61
62 else:
63 main.log.warn("Skipped pulling onos and Skipped building ONOS")
64
65 main.apps = main.params['ENV']['cellApps']
66 main.BENCHUser = main.params['BENCH']['user']
67 main.BENCHIp = main.params['BENCH']['ip1']
68 main.MN1Ip = main.params['MN']['ip1']
69 main.maxNodes = int(main.params['max'])
70 main.skipMvn = main.params['TEST']['skipCleanInstall']
71 main.cellName = main.params['ENV']['cellName']
72 main.scale = (main.params['SCALE']).split(",")
73 main.dbFileName = main.params['DATABASE']['file']
74 main.timeout = int(main.params['SLEEP']['timeout'])
75 main.startUpSleep = int(main.params['SLEEP']['startup'])
76 main.installSleep = int(main.params['SLEEP']['install'])
77 main.verifySleep = int(main.params['SLEEP']['verify'])
78 main.verifyAttempts = int(main.params['ATTEMPTS']['verify'])
79 main.sampleSize = int(main.params['TEST']['sampleSize'])
80 main.warmUp = int(main.params['TEST']['warmUp'])
81 main.intentsList = (main.params['TEST']['intents']).split(",")
82 main.ingress = main.params['TEST']['ingress']
83 main.egress = main.params['TEST']['egress']
84 main.debug = main.params['TEST']['debug']
85 for i in range ( 0, len(main.intentsList) ):
86 main.intentsList[i] = int(main.intentsList[i])
87 # Create DataBase file
88 main.log.info( "Create Database file " + main.dbFileName )
89 resultsDB = open( main.dbFileName, "w+")
90 resultsDB.close()
91
YPZhang737d0012016-03-24 13:56:24 -070092
93 def CASE1( self, main ):
YPZhangfebf7302016-05-24 16:45:56 -070094 '''
95 clean up test environment and set up
96 '''
YPZhang737d0012016-03-24 13:56:24 -070097 import time
98
YPZhangfebf7302016-05-24 16:45:56 -070099 main.log.info( "Get ONOS cluster IP" )
100 print(main.scale)
101 main.numCtrls = int(main.scale[0])
102 main.ONOSip = []
103 main.maxNumBatch = 0
104 main.AllONOSip = main.ONOSbench.getOnosIps()
105 for i in range(main.numCtrls):
106 main.ONOSip.append(main.AllONOSip[i])
107 main.log.info(main.ONOSip)
108 main.CLIs=[]
109 main.log.info("Creating list of ONOS cli handles")
110 for i in range(main.numCtrls):
111 main.CLIs.append(getattr(main, 'ONOS%scli' % (i + 1)))
YPZhang737d0012016-03-24 13:56:24 -0700112
YPZhangfebf7302016-05-24 16:45:56 -0700113 if not main.CLIs:
114 main.log.error("Failed to create the list of ONOS cli handles")
115 main.cleanup()
116 main.exit()
YPZhang737d0012016-03-24 13:56:24 -0700117
YPZhangfebf7302016-05-24 16:45:56 -0700118 main.commit = main.ONOSbench.getVersion(report=True)
119 main.commit = main.commit.split(" ")[1]
120 main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls)
121 main.log.info("Safety check, killing all ONOS processes" +
122 " before initiating environment setup")
YPZhang737d0012016-03-24 13:56:24 -0700123
YPZhangfebf7302016-05-24 16:45:56 -0700124 for i in range(main.numCtrls):
125 main.ONOSbench.onosDie(main.ONOSip[i])
YPZhang737d0012016-03-24 13:56:24 -0700126
YPZhangfebf7302016-05-24 16:45:56 -0700127 main.log.info("NODE COUNT = %s" % main.numCtrls)
YPZhang737d0012016-03-24 13:56:24 -0700128
YPZhangfebf7302016-05-24 16:45:56 -0700129 #tempOnosIp = []
130 #for i in range(main.numCtrls):
131 # tempOnosIp.append(main.AllONOSip[i])
132 #print(tempOnosIp)
133 main.ONOSbench.createCellFile(main.ONOSbench.ip_address,
134 main.cellName,
135 main.MN1Ip,
136 main.apps,
137 main.ONOSip )
138 main.step("Apply cell to environment")
139 cellResult = main.ONOSbench.setCell(main.cellName)
140 verifyResult = main.ONOSbench.verifyCell()
141 stepResult = cellResult and verifyResult
142 utilities.assert_equals(expect=main.TRUE,
143 actual=stepResult,
144 onpass="Successfully applied cell to " + \
145 "environment",
146 onfail="Failed to apply cell to environment ")
YPZhang737d0012016-03-24 13:56:24 -0700147
YPZhangfebf7302016-05-24 16:45:56 -0700148 main.step("Creating ONOS package")
YPZhang737d0012016-03-24 13:56:24 -0700149 packageResult = main.ONOSbench.onosPackage()
YPZhangfebf7302016-05-24 16:45:56 -0700150 stepResult = packageResult
151 utilities.assert_equals(expect=main.TRUE,
152 actual=stepResult,
153 onpass="Successfully created ONOS package",
154 onfail="Failed to create ONOS package")
YPZhang737d0012016-03-24 13:56:24 -0700155
YPZhangfebf7302016-05-24 16:45:56 -0700156 main.step("Uninstall ONOS package on all Nodes")
157 uninstallResult = main.TRUE
158 for i in range(int(main.numCtrls)):
159 main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i])
160 u_result = main.ONOSbench.onosUninstall(main.ONOSip[i])
161 utilities.assert_equals(expect=main.TRUE, actual=u_result,
162 onpass="Test step PASS",
163 onfail="Test step FAIL")
164 uninstallResult = (uninstallResult and u_result)
YPZhang737d0012016-03-24 13:56:24 -0700165
YPZhangfebf7302016-05-24 16:45:56 -0700166 main.step("Install ONOS package on all Nodes")
167 installResult = main.TRUE
168 for i in range(int(main.numCtrls)):
169 main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i])
170 i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i])
171 utilities.assert_equals(expect=main.TRUE, actual=i_result,
172 onpass="Test step PASS",
173 onfail="Test step FAIL")
174 installResult = installResult and i_result
YPZhang737d0012016-03-24 13:56:24 -0700175
YPZhangfebf7302016-05-24 16:45:56 -0700176 main.step("Verify ONOS nodes UP status")
177 statusResult = main.TRUE
178 for i in range(int(main.numCtrls)):
179 main.log.info("ONOS Node " + main.ONOSip[i] + " status:")
180 onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
181 utilities.assert_equals(expect=main.TRUE, actual=onos_status,
182 onpass="Test step PASS",
183 onfail="Test step FAIL")
184 statusResult = (statusResult and onos_status)
185 time.sleep(2)
186 main.step("Start ONOS CLI on all nodes")
187 cliResult = main.TRUE
188 main.log.step(" Start ONOS cli using thread ")
189 startCliResult = main.TRUE
190 pool = []
191 main.threadID=0
192 for i in range(int(main.numCtrls)):
193 t = main.Thread(target=main.CLIs[i].startOnosCli,
194 threadID=main.threadID,
195 name="startOnosCli",
196 args=[main.ONOSip[i]],
197 kwargs={"onosStartTimeout": main.timeout})
198 pool.append(t)
199 t.start()
200 main.threadID = main.threadID + 1
201 for t in pool:
202 t.join()
203 startCliResult = startCliResult and t.result
204 time.sleep(main.startUpSleep)
YPZhang737d0012016-03-24 13:56:24 -0700205
YPZhangfebf7302016-05-24 16:45:56 -0700206 # configure apps
207 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=8)
208 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="reroute")
209 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
210 main.CLIs[0].setCfg("org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled", value="false")
211 main.CLIs[0].setCfg("org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator",
212 "useFlowObjectives", value="true")
213 time.sleep(main.startUpSleep)
YPZhang737d0012016-03-24 13:56:24 -0700214
YPZhangfebf7302016-05-24 16:45:56 -0700215 # Balance Master
216 main.CLIs[0].balanceMasters()
217 if len(main.ONOSip) > 1:
218 main.CLIs[0].deviceRole("null:0000000000000003", main.ONOSip[0])
219 main.CLIs[0].deviceRole("null:0000000000000004", main.ONOSip[0])
YPZhang737d0012016-03-24 13:56:24 -0700220
YPZhangfebf7302016-05-24 16:45:56 -0700221 def CASE2( self, main):
YPZhang737d0012016-03-24 13:56:24 -0700222 import time
223 import numpy
224 import datetime
YPZhangfebf7302016-05-24 16:45:56 -0700225 import json
226 # from scipy import stats
YPZhang737d0012016-03-24 13:56:24 -0700227
228 ts = time.time()
YPZhangfebf7302016-05-24 16:45:56 -0700229 print( main.intentsList )
230 for batchSize in main.intentsList:
231 main.log.report("Intent Batch size: " + str(batchSize) + "\n ")
232 main.LatencyList = []
233 for run in range( 0, (main.warmUp + main.sampleSize) ):
234 if run >= main.warmUp:
235 main.log.info( "================================================" )
236 main.log.info( "Starting test iteration " + str(run - main.warmUp) )
237 main.log.info( "================================================" )
YPZhang737d0012016-03-24 13:56:24 -0700238 else:
YPZhangfebf7302016-05-24 16:45:56 -0700239 main.log.info( "====================Warm Up=====================" )
YPZhang737d0012016-03-24 13:56:24 -0700240
YPZhangfebf7302016-05-24 16:45:56 -0700241 # push intents
242 main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
243 offset=1,options="-i",timeout=main.timeout)
YPZhang737d0012016-03-24 13:56:24 -0700244
YPZhangfebf7302016-05-24 16:45:56 -0700245 # check links and flows
246 k = 0
247 verify = main.FALSE
248 linkCheck = 0
249 flowsCheck = 0
250 while k <= main.verifyAttempts:
251 time.sleep( main.verifySleep )
252 summary = json.loads( main.CLIs[0].summary(timeout=main.timeout) )
253 linkCheck = summary.get("links")
254 flowsCheck = summary.get("flows")
255 if linkCheck == 16 and flowsCheck == batchSize*7:
256 main.log.info( "links: {}, flows: {} ".format(linkCheck,flowsCheck) )
257 verify = main.TRUE
258 break
259 k += 1
260 if not verify:
261 main.log.warn( "Links or flows number are not match!")
262 main.log.warn( "links: {}, flows: {} ".format(linkCheck, flowsCheck) )
263 continue
YPZhang737d0012016-03-24 13:56:24 -0700264
YPZhangfebf7302016-05-24 16:45:56 -0700265 # Bring link down
266 main.CLIs[0].link("0000000000000004/1", "0000000000000003/2", "down",
267 timeout=main.timeout, showResponse=False )
268 verify = main.FALSE
269 k = 0
270 topoManagerLog = ""
271 while k <= main.verifyAttempts:
272 time.sleep(main.verifySleep)
273 summary = json.loads( main.CLIs[0].summary(timeout=main.timeout) )
274 linkCheck = summary.get("links")
275 flowsCheck = summary.get("flows")
276 if linkCheck == 14:
277 main.log.info( "links: {}, flows: {} ".format(linkCheck, flowsCheck) )
278 verify = main.TRUE
279 break
280 k += 1
281 if not verify:
282 main.log.warn( "Links number are not match in TopologyManager log!" )
283 main.log.warn( topoManagerLog )
284 continue
YPZhang737d0012016-03-24 13:56:24 -0700285
YPZhangfebf7302016-05-24 16:45:56 -0700286 try:
287 # expect twice to clean the pexpect buffer
288 main.ONOSbench.handle.sendline("")
289 main.ONOSbench.handle.expect("\$")
290 main.ONOSbench.handle.expect("\$")
291 # send line by using bench, can't use driver because pexpect buffer problem
292 cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
293 main.ONOSbench.handle.sendline(cmd)
294 time.sleep(1)
295 main.ONOSbench.handle.expect(":~")
296 topoManagerLog = main.ONOSbench.handle.before
297 topoManagerLogTemp = topoManagerLog.splitlines()
298 # To make sure we get correct topology log
299 for lines in topoManagerLogTemp:
300 if "creationTime" in lines:
301 topoManagerLog = lines
302 main.log.info("Topology Manager log:")
303 print(topoManagerLog)
304 cutTimestamp = float(topoManagerLog.split("creationTime=")[1].split(",")[0])
305 except:
306 main.log.error("Topology Log is not correct!")
307 print(topoManagerLog)
308 # If we got wrong Topology log, we should skip this iteration, and continue for next one
309 continue
310
311 installedTemp = []
312 time.sleep(1)
313 for cli in main.CLIs:
314 tempJson = json.loads( cli.intentsEventsMetrics() )
315 Installedtime = tempJson.get('intentInstalledTimestamp').get('value')
316 installedTemp.append(float(Installedtime))
317 for i in range(0, len(installedTemp)):
318 main.log.info("ONOS Node {} Installed Time stemp: {}".format((i+1),installedTemp[i]))
319 maxInstallTime = float( max(installedTemp) )
320 if run >= main.warmUp:
321 main.log.info( "Installed time stemp: {0:f}".format( maxInstallTime ) )
322 main.log.info("CutTimestamp: {0:f}".format( cutTimestamp) )
323 # Both timeStemps are milliseconds
324 main.log.info( "Latency: {0:f}".format( float(maxInstallTime-cutTimestamp)) )
325 main.LatencyList.append(float(maxInstallTime-cutTimestamp))
326
327 # Verify Summary after we bring up link, and withdrawn intents
328 main.CLIs[0].link( "0000000000000004/1", "0000000000000003/2", "up",
329 timeout=main.timeout )
330 k = 0
331 verify = main.FALSE
332 linkCheck = 0
333 flowsCheck = 0
334 while k <= main.verifyAttempts:
335 time.sleep(main.verifySleep)
336 main.CLIs[0].removeAllIntents( purge=True, sync=True, timeout=main.timeout )
337 time.sleep(1)
338 main.CLIs[0].purgeWithdrawnIntents()
339 summary = json.loads( main.CLIs[0].summary() )
340 linkCheck = summary.get("links")
341 flowsCheck = summary.get("flows")
342 intentCheck = summary.get("intents")
343 if linkCheck == 16 and flowsCheck == 0 and intentCheck == 0:
344 main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
345 verify = main.TRUE
346 break
347 k += 1
348 if not verify:
349 main.log.error("links, flows, or intents are not correct!")
350 main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
351 continue
352
353 aveLatency=0
354 stdLatency=0
355 aveLatency = numpy.average(main.LatencyList)
356 stdLatency = numpy.std(main.LatencyList)
357 main.log.report("Scale: " + str(main.numCtrls) + " \tIntent batch: " + str(batchSize))
358 main.log.report("Latency average:................" + str(aveLatency))
359 main.log.report("Latency standard deviation:....." + str(stdLatency))
YPZhang737d0012016-03-24 13:56:24 -0700360 main.log.report("________________________________________________________")
361
YPZhangfebf7302016-05-24 16:45:56 -0700362 resultsDB = open(main.dbFileName, "a")
363 resultsDB.write("'" + main.commit + "',")
364 resultsDB.write(str(main.numCtrls) + ",")
365 resultsDB.write(str(batchSize) + ",")
366 resultsDB.write(str(aveLatency) + ",")
367 resultsDB.write(str(stdLatency) + "\n")
YPZhang737d0012016-03-24 13:56:24 -0700368 resultsDB.close()
YPZhangfebf7302016-05-24 16:45:56 -0700369 del main.scale[0]