blob: 730875e6326c352bb554204ff00987b4ae433275 [file] [log] [blame]
YPZhangb34b7e12016-06-14 14:28:19 -07001"""
2SCPFintentInstallWithdrawLat:
3 - Test the latency of intent installed and withdrawn
4 - Use Push-test-intents command to push intents
5 - Use Null provider with 7 devices and linear topology
6 - Always push intents between 1/6 and 7/5
7 - The batch size is defined in parm file. (default 1,100,1000)
cameron@onlab.us41c16f52015-07-08 15:40:28 -07008
YPZhangb34b7e12016-06-14 14:28:19 -07009 yunpeng@onlab.us
10"""
cameron@onlab.us41c16f52015-07-08 15:40:28 -070011class SCPFintentInstallWithdrawLat:
cameron@onlab.us41c16f52015-07-08 15:40:28 -070012 def __init__( self ):
13 self.default = ''
14
YPZhangb34b7e12016-06-14 14:28:19 -070015 def CASE0( self, main ):
16 '''
17 - GIT
18 - BUILDING ONOS
19 Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
20 This step is usually skipped. Because in a Jenkins driven automated
21 test env. We want Jenkins jobs to pull&build for flexibility to handle
22 different versions of ONOS.
23 - Construct tests variables
24 '''
25 gitPull = main.params['GIT']['gitPull']
26 gitBranch = main.params['GIT']['gitBranch']
27
28 main.case("Pull onos branch and build onos on Teststation.")
29
30 if gitPull == 'True':
31 main.step("Git Checkout ONOS branch: " + gitBranch)
32 stepResult = main.ONOSbench.gitCheckout(branch=gitBranch)
33 utilities.assert_equals(expect=main.TRUE,
34 actual=stepResult,
35 onpass="Successfully checkout onos branch.",
36 onfail="Failed to checkout onos branch. Exiting test...")
37 if not stepResult: main.exit()
38
39 main.step("Git Pull on ONOS branch:" + gitBranch)
40 stepResult = main.ONOSbench.gitPull()
41 utilities.assert_equals(expect=main.TRUE,
42 actual=stepResult,
43 onpass="Successfully pull onos. ",
44 onfail="Failed to pull onos. Exiting test ...")
45 if not stepResult: main.exit()
46
47 main.step("Building ONOS branch: " + gitBranch)
48 stepResult = main.ONOSbench.cleanInstall(skipTest=True)
49 utilities.assert_equals(expect=main.TRUE,
50 actual=stepResult,
51 onpass="Successfully build onos.",
52 onfail="Failed to build onos. Exiting test...")
53 if not stepResult: main.exit()
54
55 else:
56 main.log.warn("Skipped pulling onos and Skipped building ONOS")
57
58 main.apps = main.params['ENV']['cellApps']
59 main.BENCHUser = main.params['BENCH']['user']
60 main.BENCHIp = main.params['BENCH']['ip1']
61 main.MN1Ip = main.params['MN']['ip1']
62 main.maxNodes = int(main.params['max'])
63 main.cellName = main.params['ENV']['cellName']
64 main.scale = (main.params['SCALE']).split(",")
65 main.dbFileName = main.params['DATABASE']['file']
66 main.timeout = int(main.params['SLEEP']['timeout'])
67 main.startUpSleep = int(main.params['SLEEP']['startup'])
68 main.installSleep = int(main.params['SLEEP']['install'])
69 main.verifySleep = int(main.params['SLEEP']['verify'])
70 main.verifyAttempts = int(main.params['ATTEMPTS']['verify'])
71 main.sampleSize = int(main.params['TEST']['sampleSize'])
72 main.warmUp = int(main.params['TEST']['warmUp'])
73 main.intentsList = (main.params['TEST']['intents']).split(",")
74 main.ingress = main.params['TEST']['ingress']
75 main.egress = main.params['TEST']['egress']
76 main.debug = main.params['TEST']['debug']
77 for i in range(0, len(main.intentsList)):
78 main.intentsList[i] = int(main.intentsList[i])
79 # Create DataBase file
80 main.log.info("Create Database file " + main.dbFileName)
81 resultsDB = open(main.dbFileName, "w+")
82 resultsDB.close()
83
Jon Hall4ba53f02015-07-29 13:07:41 -070084 def CASE1( self, main ):
YPZhangb34b7e12016-06-14 14:28:19 -070085 # Clean up test environment and set up
Jon Hall4ba53f02015-07-29 13:07:41 -070086 import time
YPZhangb34b7e12016-06-14 14:28:19 -070087 main.log.info("Get ONOS cluster IP")
88 print(main.scale)
89 main.numCtrls = int(main.scale[0])
90 main.ONOSip = []
91 main.maxNumBatch = 0
92 main.AllONOSip = main.ONOSbench.getOnosIps()
93 for i in range(main.numCtrls):
94 main.ONOSip.append(main.AllONOSip[i])
95 main.log.info(main.ONOSip)
96 main.CLIs = []
97 main.log.info("Creating list of ONOS cli handles")
98 for i in range(main.numCtrls):
99 main.CLIs.append(getattr(main, 'ONOS%scli' % (i + 1)))
Jon Hall4ba53f02015-07-29 13:07:41 -0700100
YPZhangb34b7e12016-06-14 14:28:19 -0700101 if not main.CLIs:
102 main.log.error("Failed to create the list of ONOS cli handles")
103 main.cleanup()
104 main.exit()
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700105
YPZhangb34b7e12016-06-14 14:28:19 -0700106 main.commit = main.ONOSbench.getVersion(report=True)
107 main.commit = main.commit.split(" ")[1]
108 main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls)
109 main.log.info("Safety check, killing all ONOS processes" +
110 " before initiating environment setup")
Jon Hall4ba53f02015-07-29 13:07:41 -0700111
YPZhangb34b7e12016-06-14 14:28:19 -0700112 for i in range(main.numCtrls):
113 main.ONOSbench.onosDie(main.ONOSip[i])
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700114
YPZhangb34b7e12016-06-14 14:28:19 -0700115 main.log.info("NODE COUNT = %s" % main.numCtrls)
116 main.ONOSbench.createCellFile(main.ONOSbench.ip_address,
117 main.cellName,
118 main.MN1Ip,
119 main.apps,
120 main.ONOSip)
121 main.step("Apply cell to environment")
122 cellResult = main.ONOSbench.setCell(main.cellName)
123 verifyResult = main.ONOSbench.verifyCell()
124 stepResult = cellResult and verifyResult
125 utilities.assert_equals(expect=main.TRUE,
126 actual=stepResult,
127 onpass="Successfully applied cell to " + \
128 "environment",
129 onfail="Failed to apply cell to environment ")
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700130
YPZhangb34b7e12016-06-14 14:28:19 -0700131 main.step("Creating ONOS package")
Jon Hall4ba53f02015-07-29 13:07:41 -0700132 packageResult = main.ONOSbench.onosPackage()
YPZhangb34b7e12016-06-14 14:28:19 -0700133 stepResult = packageResult
134 utilities.assert_equals(expect=main.TRUE,
135 actual=stepResult,
136 onpass="Successfully created ONOS package",
137 onfail="Failed to create ONOS package")
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700138
YPZhangb34b7e12016-06-14 14:28:19 -0700139 main.step("Uninstall ONOS package on all Nodes")
140 uninstallResult = main.TRUE
141 for i in range(int(main.numCtrls)):
142 main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i])
143 u_result = main.ONOSbench.onosUninstall(main.ONOSip[i])
144 utilities.assert_equals(expect=main.TRUE, actual=u_result,
145 onpass="Test step PASS",
146 onfail="Test step FAIL")
147 uninstallResult = (uninstallResult and u_result)
Jon Hall4ba53f02015-07-29 13:07:41 -0700148
YPZhangb34b7e12016-06-14 14:28:19 -0700149 main.step("Install ONOS package on all Nodes")
150 installResult = main.TRUE
151 for i in range(int(main.numCtrls)):
152 main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i])
153 i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i])
154 utilities.assert_equals(expect=main.TRUE, actual=i_result,
155 onpass="Test step PASS",
156 onfail="Test step FAIL")
157 installResult = installResult and i_result
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700158
YPZhangb34b7e12016-06-14 14:28:19 -0700159 main.step("Verify ONOS nodes UP status")
160 statusResult = main.TRUE
161 for i in range(int(main.numCtrls)):
162 main.log.info("ONOS Node " + main.ONOSip[i] + " status:")
163 onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
164 utilities.assert_equals(expect=main.TRUE, actual=onos_status,
165 onpass="Test step PASS",
166 onfail="Test step FAIL")
167 statusResult = (statusResult and onos_status)
168 time.sleep(2)
169 main.step("Start ONOS CLI on all nodes")
170 cliResult = main.TRUE
171 main.log.step(" Start ONOS cli using thread ")
172 startCliResult = main.TRUE
173 pool = []
174 main.threadID = 0
175 for i in range(int(main.numCtrls)):
176 t = main.Thread(target=main.CLIs[i].startOnosCli,
177 threadID=main.threadID,
178 name="startOnosCli",
179 args=[main.ONOSip[i]],
180 kwargs={"onosStartTimeout": main.timeout})
181 pool.append(t)
182 t.start()
183 main.threadID = main.threadID + 1
184 for t in pool:
185 t.join()
186 startCliResult = startCliResult and t.result
187 time.sleep(main.startUpSleep)
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700188
YPZhangb34b7e12016-06-14 14:28:19 -0700189 # configure apps
190 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=7)
191 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="linear")
192 main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
YPZhang325822f2016-06-17 16:01:45 -0700193 main.CLIs[0].setCfg("org.onosproject.net.intent.impl.IntentManager", "skipReleaseResourcesOnWithdrawal", value="true")
YPZhangb34b7e12016-06-14 14:28:19 -0700194 time.sleep(main.startUpSleep)
Jon Hall4ba53f02015-07-29 13:07:41 -0700195
YPZhangb34b7e12016-06-14 14:28:19 -0700196 # balanceMasters
197 main.CLIs[0].balanceMasters()
You Wang3b148ea2016-06-20 20:33:42 -0700198 time.sleep(main.startUpSleep)
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700199
200 def CASE2( self, main ):
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700201 import time
202 import numpy
YPZhangb34b7e12016-06-14 14:28:19 -0700203 import json
204 print(main.intentsList)
205 for batchSize in main.intentsList:
206 main.log.report("Intent Batch size: {}".format(batchSize))
207 main.installLatList = []
208 main.withdrawLatList = []
209 validrun = 0
210 invalidrun = 0
211 # we use two variables to control the iteration
212 while validrun <= main.warmUp + main.sampleSize and invalidrun < 20:
213 if validrun >= main.warmUp:
214 main.log.info("================================================")
215 main.log.info("Starting test iteration " + str(validrun - main.warmUp))
216 main.log.info("Total test iteration: " + str(invalidrun + validrun))
217 main.log.info("================================================")
218 else:
219 main.log.info("====================Warm Up=====================")
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700220
YPZhangb34b7e12016-06-14 14:28:19 -0700221 # push intents
222 installResult = main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
223 offset=1, options="-i", timeout=main.timeout,
224 getResponse=True)
225 if type(installResult) is str:
226 if "Failure" in installResult:
227 main.log.error("Install Intents failure, ignore this iteration.")
228 if validrun < main.warmUp:
229 validrun += 1
230 continue
231 else:
232 invalidrun += 1
233 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700234
YPZhangb34b7e12016-06-14 14:28:19 -0700235 try:
236 latency = int(installResult.split()[5])
237 main.log.info(installResult)
238 except:
239 main.log.error("Failed to get latency, ignore this iteration.")
240 main.log.error("Response from ONOS:")
241 print(installResult)
242 if validrun < main.warmUp:
243 validrun += 1
244 continue
245 else:
246 invalidrun += 1
247 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700248
YPZhangb34b7e12016-06-14 14:28:19 -0700249 if validrun >= main.warmUp:
250 main.installLatList.append(latency)
251 else:
252 invalidrun += 1
253 continue
254 time.sleep(2)
255 # Withdraw Intents
256 withdrawResult = main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
257 offset=1, options="-w", timeout=main.timeout,
258 getResponse=True)
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700259
YPZhangb34b7e12016-06-14 14:28:19 -0700260 if type(withdrawResult) is str:
261 if "Failure" in withdrawResult:
262 main.log.error("withdraw Intents failure, ignore this iteration.")
263 if validrun < main.warmUp:
264 validrun += 1
265 continue
266 else:
267 invalidrun += 1
268 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700269
YPZhangb34b7e12016-06-14 14:28:19 -0700270 try:
271 latency = int(withdrawResult.split()[5])
272 main.log.info(withdrawResult)
273 except:
274 main.log.error("Failed to get latency, ignore this iteration.")
275 main.log.error("Response from ONOS:")
276 print(withdrawResult)
277 if validrun < main.warmUp:
278 validrun += 1
279 continue
280 else:
281 invalidrun += 1
282 continue
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700283
YPZhangb34b7e12016-06-14 14:28:19 -0700284 if validrun >= main.warmUp:
285 main.withdrawLatList.append(latency)
286 else:
287 invalidrun += 1
288 continue
289 time.sleep(2)
290 main.CLIs[0].purgeWithdrawnIntents()
291 validrun += 1
292 installave = numpy.average(main.installLatList)
293 installstd = numpy.std(main.installLatList)
294 withdrawave = numpy.average(main.withdrawLatList)
295 withdrawstd = numpy.std(main.withdrawLatList)
296 # log report
cameron@onlab.us41c16f52015-07-08 15:40:28 -0700297 main.log.report("----------------------------------------------------")
YPZhangb34b7e12016-06-14 14:28:19 -0700298 main.log.report("Scale: " + str(main.numCtrls))
299 main.log.report("Intent batch: " + str(batchSize))
300 main.log.report("Install average: {} std: {}".format(installave, installstd))
301 main.log.report("Withdraw average: {} std: {}".format(withdrawave, withdrawstd))
302 # write result to database file
303 if not (numpy.isnan(installave) or numpy.isnan(installstd) or\
304 numpy.isnan(withdrawstd) or numpy.isnan(withdrawave)):
305 databaseString = "'" + main.commit + "',"
306 databaseString += str(main.numCtrls) + ","
307 databaseString += str(batchSize) + ","
308 databaseString += str(installave) + ","
309 databaseString += str(installstd) + ","
310 databaseString += str(withdrawave) + ","
311 databaseString += str(withdrawstd) + "\n"
312 resultsDB = open(main.dbFileName, "a")
313 resultsDB.write(databaseString)
314 resultsDB.close()
315 del main.scale[0]