blob: 0b8a465198804ff348da3bb8e5c53be8e6b7a89a [file] [log] [blame]
YPZhangb34b7e12016-06-14 14:28:19 -07001"""
2SCPFintentInstallWithdrawLat:
3 - Test the latency of intent installed and withdrawn
4 - Use Push-test-intents command to push intents
5 - Use Null provider with 7 devices and linear topology
6 - Always push intents between 1/6 and 7/5
7 - The batch size is defined in parm file. (default 1,100,1000)
cameron@onlab.us41c16f52015-07-08 15:40:28 -07008
YPZhangb34b7e12016-06-14 14:28:19 -07009 yunpeng@onlab.us
10"""
cameron@onlab.us41c16f52015-07-08 15:40:28 -070011class SCPFintentInstallWithdrawLat:
cameron@onlab.us41c16f52015-07-08 15:40:28 -070012 def __init__( self ):
13 self.default = ''
14
YPZhangb34b7e12016-06-14 14:28:19 -070015 def CASE0( self, main ):
16 '''
17 - GIT
18 - BUILDING ONOS
19 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
20 This step is usually skipped. Because in a Jenkins driven automated
21 test env. We want Jenkins jobs to pull&build for flexibility to handle
22 different versions of ONOS.
23 - Construct tests variables
24 '''
25 gitPull = main.params['GIT']['gitPull']
26 gitBranch = main.params['GIT']['gitBranch']
27
28 main.case("Pull onos branch and build onos on Teststation.")
29
30 if gitPull == 'True':
31 main.step("Git Checkout ONOS branch: " + gitBranch)
32 stepResult = main.ONOSbench.gitCheckout(branch=gitBranch)
33 utilities.assert_equals(expect=main.TRUE,
34 actual=stepResult,
35 onpass="Successfully checkout onos branch.",
36 onfail="Failed to checkout onos branch. Exiting test...")
37 if not stepResult: main.exit()
38
39 main.step("Git Pull on ONOS branch:" + gitBranch)
40 stepResult = main.ONOSbench.gitPull()
41 utilities.assert_equals(expect=main.TRUE,
42 actual=stepResult,
43 onpass="Successfully pull onos. ",
44 onfail="Failed to pull onos. Exiting test ...")
45 if not stepResult: main.exit()
46
47 main.step("Building ONOS branch: " + gitBranch)
48 stepResult = main.ONOSbench.cleanInstall(skipTest=True)
49 utilities.assert_equals(expect=main.TRUE,
50 actual=stepResult,
51 onpass="Successfully build onos.",
52 onfail="Failed to build onos. Exiting test...")
53 if not stepResult: main.exit()
54
55 else:
56 main.log.warn("Skipped pulling onos and Skipped building ONOS")
57
58 main.apps = main.params['ENV']['cellApps']
59 main.BENCHUser = main.params['BENCH']['user']
60 main.BENCHIp = main.params['BENCH']['ip1']
61 main.MN1Ip = main.params['MN']['ip1']
62 main.maxNodes = int(main.params['max'])
63 main.cellName = main.params['ENV']['cellName']
64 main.scale = (main.params['SCALE']).split(",")
YPZhangb34b7e12016-06-14 14:28:19 -070065 main.timeout = int(main.params['SLEEP']['timeout'])
66 main.startUpSleep = int(main.params['SLEEP']['startup'])
67 main.installSleep = int(main.params['SLEEP']['install'])
68 main.verifySleep = int(main.params['SLEEP']['verify'])
69 main.verifyAttempts = int(main.params['ATTEMPTS']['verify'])
70 main.sampleSize = int(main.params['TEST']['sampleSize'])
71 main.warmUp = int(main.params['TEST']['warmUp'])
72 main.intentsList = (main.params['TEST']['intents']).split(",")
73 main.ingress = main.params['TEST']['ingress']
74 main.egress = main.params['TEST']['egress']
75 main.debug = main.params['TEST']['debug']
YPZhange6ef82a2016-07-05 16:48:15 -070076 main.flowObj = main.params['TEST']['flowObj']
77
78 if main.flowObj == "True":
79 main.flowObj = True
80 main.dbFileName = main.params['DATABASE']['dbFlowObj']
81 else:
82 main.flowObj = False
83 main.dbFileName = main.params['DATABASE']['dbName']
84
YPZhangb34b7e12016-06-14 14:28:19 -070085 for i in range(0, len(main.intentsList)):
86 main.intentsList[i] = int(main.intentsList[i])
87 # Create DataBase file
88 main.log.info("Create Database file " + main.dbFileName)
89 resultsDB = open(main.dbFileName, "w+")
90 resultsDB.close()
91
Jon Hall4ba53f02015-07-29 13:07:41 -070092 def CASE1( self, main ):
YPZhangb34b7e12016-06-14 14:28:19 -070093 # Clean up test environment and set up
Jon Hall4ba53f02015-07-29 13:07:41 -070094 import time
YPZhangb34b7e12016-06-14 14:28:19 -070095 main.log.info("Get ONOS cluster IP")
96 print(main.scale)
97 main.numCtrls = int(main.scale[0])
98 main.ONOSip = []
99 main.maxNumBatch = 0
100 main.AllONOSip = main.ONOSbench.getOnosIps()
101 for i in range(main.numCtrls):
102 main.ONOSip.append(main.AllONOSip[i])
103 main.log.info(main.ONOSip)
104 main.CLIs = []
105 main.log.info("Creating list of ONOS cli handles")
106 for i in range(main.numCtrls):
107 main.CLIs.append(getattr(main, 'ONOS%scli' % (i + 1)))
Jon Hall4ba53f02015-07-29 13:07:41 -0700108
YPZhangb34b7e12016-06-14 14:28:19 -0700109 if not main.CLIs:
110 main.log.error("Failed to create the list of ONOS cli handles")
111 main.cleanup()
112 main.exit()
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700113
YPZhangb34b7e12016-06-14 14:28:19 -0700114 main.commit = main.ONOSbench.getVersion(report=True)
115 main.commit = main.commit.split(" ")[1]
116 main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls)
117 main.log.info("Safety check, killing all ONOS processes" +
118 " before initiating environment setup")
Jon Hall4ba53f02015-07-29 13:07:41 -0700119
YPZhangb34b7e12016-06-14 14:28:19 -0700120 for i in range(main.numCtrls):
121 main.ONOSbench.onosDie(main.ONOSip[i])
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700122
YPZhangb34b7e12016-06-14 14:28:19 -0700123 main.log.info("NODE COUNT = %s" % main.numCtrls)
124 main.ONOSbench.createCellFile(main.ONOSbench.ip_address,
125 main.cellName,
126 main.MN1Ip,
127 main.apps,
128 main.ONOSip)
129 main.step("Apply cell to environment")
130 cellResult = main.ONOSbench.setCell(main.cellName)
131 verifyResult = main.ONOSbench.verifyCell()
132 stepResult = cellResult and verifyResult
133 utilities.assert_equals(expect=main.TRUE,
134 actual=stepResult,
135 onpass="Successfully applied cell to " + \
136 "environment",
137 onfail="Failed to apply cell to environment ")
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700138
YPZhangb34b7e12016-06-14 14:28:19 -0700139 main.step("Creating ONOS package")
Jon Hallbd60ea02016-08-23 10:03:59 -0700140 packageResult = main.ONOSbench.buckBuild()
YPZhangb34b7e12016-06-14 14:28:19 -0700141 stepResult = packageResult
142 utilities.assert_equals(expect=main.TRUE,
143 actual=stepResult,
144 onpass="Successfully created ONOS package",
145 onfail="Failed to create ONOS package")
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700146
YPZhangb34b7e12016-06-14 14:28:19 -0700147 main.step("Uninstall ONOS package on all Nodes")
148 uninstallResult = main.TRUE
149 for i in range(int(main.numCtrls)):
150 main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i])
151 u_result = main.ONOSbench.onosUninstall(main.ONOSip[i])
152 utilities.assert_equals(expect=main.TRUE, actual=u_result,
153 onpass="Test step PASS",
154 onfail="Test step FAIL")
155 uninstallResult = (uninstallResult and u_result)
Jon Hall4ba53f02015-07-29 13:07:41 -0700156
YPZhangb34b7e12016-06-14 14:28:19 -0700157 main.step("Install ONOS package on all Nodes")
158 installResult = main.TRUE
159 for i in range(int(main.numCtrls)):
160 main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i])
161 i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i])
162 utilities.assert_equals(expect=main.TRUE, actual=i_result,
163 onpass="Test step PASS",
164 onfail="Test step FAIL")
165 installResult = installResult and i_result
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700166
YPZhangb34b7e12016-06-14 14:28:19 -0700167 main.step("Verify ONOS nodes UP status")
168 statusResult = main.TRUE
169 for i in range(int(main.numCtrls)):
170 main.log.info("ONOS Node " + main.ONOSip[i] + " status:")
171 onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
172 utilities.assert_equals(expect=main.TRUE, actual=onos_status,
173 onpass="Test step PASS",
174 onfail="Test step FAIL")
175 statusResult = (statusResult and onos_status)
176 time.sleep(2)
177 main.step("Start ONOS CLI on all nodes")
178 cliResult = main.TRUE
Jon Hall6509dbf2016-06-21 17:01:17 -0700179 main.step(" Start ONOS cli using thread ")
YPZhangb34b7e12016-06-14 14:28:19 -0700180 startCliResult = main.TRUE
181 pool = []
182 main.threadID = 0
183 for i in range(int(main.numCtrls)):
184 t = main.Thread(target=main.CLIs[i].startOnosCli,
185 threadID=main.threadID,
186 name="startOnosCli",
187 args=[main.ONOSip[i]],
188 kwargs={"onosStartTimeout": main.timeout})
189 pool.append(t)
190 t.start()
191 main.threadID = main.threadID + 1
192 for t in pool:
193 t.join()
194 startCliResult = startCliResult and t.result
195 time.sleep(main.startUpSleep)
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700196
YPZhangb34b7e12016-06-14 14:28:19 -0700197 # configure apps
198 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=7)
199 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="linear")
200 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
YPZhang325822f2016-06-17 16:01:45 -0700201 main.CLIs[0].setCfg("org.onosproject.net.intent.impl.IntentManager", "skipReleaseResourcesOnWithdrawal", value="true")
YPZhange6ef82a2016-07-05 16:48:15 -0700202 if main.flowObj:
203 main.CLIs[0].setCfg("org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator",
204 "useFlowObjectives", value="true")
YPZhangb34b7e12016-06-14 14:28:19 -0700205 time.sleep(main.startUpSleep)
Jon Hall4ba53f02015-07-29 13:07:41 -0700206
YPZhangb34b7e12016-06-14 14:28:19 -0700207 # balanceMasters
208 main.CLIs[0].balanceMasters()
You Wang3b148ea2016-06-20 20:33:42 -0700209 time.sleep(main.startUpSleep)
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700210
211 def CASE2( self, main ):
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700212 import time
213 import numpy
YPZhangb34b7e12016-06-14 14:28:19 -0700214 import json
215 print(main.intentsList)
216 for batchSize in main.intentsList:
217 main.log.report("Intent Batch size: {}".format(batchSize))
218 main.installLatList = []
219 main.withdrawLatList = []
220 validrun = 0
221 invalidrun = 0
222 # we use two variables to control the iteration
223 while validrun <= main.warmUp + main.sampleSize and invalidrun < 20:
224 if validrun >= main.warmUp:
225 main.log.info("================================================")
226 main.log.info("Starting test iteration " + str(validrun - main.warmUp))
227 main.log.info("Total test iteration: " + str(invalidrun + validrun))
228 main.log.info("================================================")
229 else:
230 main.log.info("====================Warm Up=====================")
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700231
YPZhangb34b7e12016-06-14 14:28:19 -0700232 # push intents
233 installResult = main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
234 offset=1, options="-i", timeout=main.timeout,
235 getResponse=True)
236 if type(installResult) is str:
237 if "Failure" in installResult:
238 main.log.error("Install Intents failure, ignore this iteration.")
239 if validrun < main.warmUp:
240 validrun += 1
241 continue
242 else:
243 invalidrun += 1
244 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700245
YPZhangb34b7e12016-06-14 14:28:19 -0700246 try:
247 latency = int(installResult.split()[5])
248 main.log.info(installResult)
249 except:
250 main.log.error("Failed to get latency, ignore this iteration.")
251 main.log.error("Response from ONOS:")
252 print(installResult)
253 if validrun < main.warmUp:
254 validrun += 1
255 continue
256 else:
257 invalidrun += 1
258 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700259
YPZhangb34b7e12016-06-14 14:28:19 -0700260 if validrun >= main.warmUp:
261 main.installLatList.append(latency)
262 else:
263 invalidrun += 1
264 continue
265 time.sleep(2)
266 # Withdraw Intents
267 withdrawResult = main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
268 offset=1, options="-w", timeout=main.timeout,
269 getResponse=True)
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700270
YPZhangb34b7e12016-06-14 14:28:19 -0700271 if type(withdrawResult) is str:
272 if "Failure" in withdrawResult:
273 main.log.error("withdraw Intents failure, ignore this iteration.")
274 if validrun < main.warmUp:
275 validrun += 1
276 continue
277 else:
278 invalidrun += 1
279 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700280
YPZhangb34b7e12016-06-14 14:28:19 -0700281 try:
282 latency = int(withdrawResult.split()[5])
283 main.log.info(withdrawResult)
284 except:
285 main.log.error("Failed to get latency, ignore this iteration.")
286 main.log.error("Response from ONOS:")
287 print(withdrawResult)
288 if validrun < main.warmUp:
289 validrun += 1
290 continue
291 else:
292 invalidrun += 1
293 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700294
YPZhangb34b7e12016-06-14 14:28:19 -0700295 if validrun >= main.warmUp:
296 main.withdrawLatList.append(latency)
297 else:
298 invalidrun += 1
299 continue
300 time.sleep(2)
301 main.CLIs[0].purgeWithdrawnIntents()
302 validrun += 1
303 installave = numpy.average(main.installLatList)
304 installstd = numpy.std(main.installLatList)
305 withdrawave = numpy.average(main.withdrawLatList)
306 withdrawstd = numpy.std(main.withdrawLatList)
307 # log report
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700308 main.log.report("----------------------------------------------------")
YPZhangb34b7e12016-06-14 14:28:19 -0700309 main.log.report("Scale: " + str(main.numCtrls))
310 main.log.report("Intent batch: " + str(batchSize))
311 main.log.report("Install average: {} std: {}".format(installave, installstd))
312 main.log.report("Withdraw average: {} std: {}".format(withdrawave, withdrawstd))
313 # write result to database file
314 if not (numpy.isnan(installave) or numpy.isnan(installstd) or\
315 numpy.isnan(withdrawstd) or numpy.isnan(withdrawave)):
316 databaseString = "'" + main.commit + "',"
317 databaseString += str(main.numCtrls) + ","
318 databaseString += str(batchSize) + ","
319 databaseString += str(installave) + ","
320 databaseString += str(installstd) + ","
321 databaseString += str(withdrawave) + ","
322 databaseString += str(withdrawstd) + "\n"
323 resultsDB = open(main.dbFileName, "a")
324 resultsDB.write(databaseString)
325 resultsDB.close()
326 del main.scale[0]