YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 1 | """ |
| 2 | SCPFintentInstallWithdrawLat: |
| 3 | - Test the latency of intent installed and withdrawn |
| 4 | - Use Push-test-intents command to push intents |
| 5 | - Use Null provider with 7 devices and linear topology |
| 6 | - Always push intents between 1/6 and 7/5 |
| 7 | - The batch size is defined in parm file. (default 1,100,1000) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 8 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 9 | yunpeng@onlab.us |
| 10 | """ |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 11 | class SCPFintentInstallWithdrawLat: |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 12 | def __init__( self ): |
| 13 | self.default = '' |
| 14 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 15 | def CASE0( self, main ): |
| 16 | ''' |
| 17 | - GIT |
| 18 | - BUILDING ONOS |
| 19 | Pull specific ONOS branch, then Build ONOS ono ONOS Bench. |
| 20 | This step is usually skipped. Because in a Jenkins driven automated |
| 21 | test env. We want Jenkins jobs to pull&build for flexibility to handle |
| 22 | different versions of ONOS. |
| 23 | - Construct tests variables |
| 24 | ''' |
| 25 | gitPull = main.params['GIT']['gitPull'] |
| 26 | gitBranch = main.params['GIT']['gitBranch'] |
| 27 | |
| 28 | main.case("Pull onos branch and build onos on Teststation.") |
| 29 | |
| 30 | if gitPull == 'True': |
| 31 | main.step("Git Checkout ONOS branch: " + gitBranch) |
| 32 | stepResult = main.ONOSbench.gitCheckout(branch=gitBranch) |
| 33 | utilities.assert_equals(expect=main.TRUE, |
| 34 | actual=stepResult, |
| 35 | onpass="Successfully checkout onos branch.", |
| 36 | onfail="Failed to checkout onos branch. Exiting test...") |
| 37 | if not stepResult: main.exit() |
| 38 | |
| 39 | main.step("Git Pull on ONOS branch:" + gitBranch) |
| 40 | stepResult = main.ONOSbench.gitPull() |
| 41 | utilities.assert_equals(expect=main.TRUE, |
| 42 | actual=stepResult, |
| 43 | onpass="Successfully pull onos. ", |
| 44 | onfail="Failed to pull onos. Exiting test ...") |
| 45 | if not stepResult: main.exit() |
| 46 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 47 | |
| 48 | else: |
| 49 | main.log.warn("Skipped pulling onos and Skipped building ONOS") |
| 50 | |
| 51 | main.apps = main.params['ENV']['cellApps'] |
| 52 | main.BENCHUser = main.params['BENCH']['user'] |
| 53 | main.BENCHIp = main.params['BENCH']['ip1'] |
| 54 | main.MN1Ip = main.params['MN']['ip1'] |
| 55 | main.maxNodes = int(main.params['max']) |
| 56 | main.cellName = main.params['ENV']['cellName'] |
| 57 | main.scale = (main.params['SCALE']).split(",") |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 58 | main.timeout = int(main.params['SLEEP']['timeout']) |
| 59 | main.startUpSleep = int(main.params['SLEEP']['startup']) |
| 60 | main.installSleep = int(main.params['SLEEP']['install']) |
| 61 | main.verifySleep = int(main.params['SLEEP']['verify']) |
| 62 | main.verifyAttempts = int(main.params['ATTEMPTS']['verify']) |
| 63 | main.sampleSize = int(main.params['TEST']['sampleSize']) |
| 64 | main.warmUp = int(main.params['TEST']['warmUp']) |
| 65 | main.intentsList = (main.params['TEST']['intents']).split(",") |
| 66 | main.ingress = main.params['TEST']['ingress'] |
| 67 | main.egress = main.params['TEST']['egress'] |
| 68 | main.debug = main.params['TEST']['debug'] |
YPZhang | e6ef82a | 2016-07-05 16:48:15 -0700 | [diff] [blame] | 69 | main.flowObj = main.params['TEST']['flowObj'] |
| 70 | |
| 71 | if main.flowObj == "True": |
| 72 | main.flowObj = True |
| 73 | main.dbFileName = main.params['DATABASE']['dbFlowObj'] |
| 74 | else: |
| 75 | main.flowObj = False |
| 76 | main.dbFileName = main.params['DATABASE']['dbName'] |
| 77 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 78 | for i in range(0, len(main.intentsList)): |
| 79 | main.intentsList[i] = int(main.intentsList[i]) |
| 80 | # Create DataBase file |
| 81 | main.log.info("Create Database file " + main.dbFileName) |
| 82 | resultsDB = open(main.dbFileName, "w+") |
| 83 | resultsDB.close() |
| 84 | |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 85 | def CASE1( self, main ): |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 86 | # Clean up test environment and set up |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 87 | import time |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 88 | main.log.info("Get ONOS cluster IP") |
| 89 | print(main.scale) |
| 90 | main.numCtrls = int(main.scale[0]) |
| 91 | main.ONOSip = [] |
| 92 | main.maxNumBatch = 0 |
| 93 | main.AllONOSip = main.ONOSbench.getOnosIps() |
| 94 | for i in range(main.numCtrls): |
| 95 | main.ONOSip.append(main.AllONOSip[i]) |
| 96 | main.log.info(main.ONOSip) |
| 97 | main.CLIs = [] |
| 98 | main.log.info("Creating list of ONOS cli handles") |
| 99 | for i in range(main.numCtrls): |
Devin Lim | 2870684 | 2017-06-08 10:23:48 -0700 | [diff] [blame] | 100 | main.CLIs.append(getattr(main, 'ONOScli%s' %(i + 1))) |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 101 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 102 | if not main.CLIs: |
| 103 | main.log.error("Failed to create the list of ONOS cli handles") |
| 104 | main.cleanup() |
| 105 | main.exit() |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 106 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 107 | main.commit = main.ONOSbench.getVersion(report=True) |
| 108 | main.commit = main.commit.split(" ")[1] |
| 109 | main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls) |
| 110 | main.log.info("Safety check, killing all ONOS processes" + |
| 111 | " before initiating environment setup") |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 112 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 113 | for i in range(main.numCtrls): |
You Wang | b98a9fa | 2017-02-15 17:27:42 -0800 | [diff] [blame] | 114 | main.ONOSbench.onosStop(main.ONOSip[i]) |
| 115 | main.ONOSbench.onosKill(main.ONOSip[i]) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 116 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 117 | main.log.info("NODE COUNT = %s" % main.numCtrls) |
| 118 | main.ONOSbench.createCellFile(main.ONOSbench.ip_address, |
| 119 | main.cellName, |
| 120 | main.MN1Ip, |
| 121 | main.apps, |
Devin Lim | 461f087 | 2017-06-05 16:49:33 -0700 | [diff] [blame] | 122 | main.ONOSip, |
Devin Lim | dc78e20 | 2017-06-09 18:30:07 -0700 | [diff] [blame] | 123 | main.ONOScli1.karafUser ) |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 124 | main.step("Apply cell to environment") |
| 125 | cellResult = main.ONOSbench.setCell(main.cellName) |
| 126 | verifyResult = main.ONOSbench.verifyCell() |
| 127 | stepResult = cellResult and verifyResult |
| 128 | utilities.assert_equals(expect=main.TRUE, |
| 129 | actual=stepResult, |
| 130 | onpass="Successfully applied cell to " + \ |
| 131 | "environment", |
| 132 | onfail="Failed to apply cell to environment ") |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 133 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 134 | main.step("Creating ONOS package") |
Jon Hall | bd60ea0 | 2016-08-23 10:03:59 -0700 | [diff] [blame] | 135 | packageResult = main.ONOSbench.buckBuild() |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 136 | stepResult = packageResult |
| 137 | utilities.assert_equals(expect=main.TRUE, |
| 138 | actual=stepResult, |
| 139 | onpass="Successfully created ONOS package", |
| 140 | onfail="Failed to create ONOS package") |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 141 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 142 | main.step("Uninstall ONOS package on all Nodes") |
| 143 | uninstallResult = main.TRUE |
| 144 | for i in range(int(main.numCtrls)): |
| 145 | main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i]) |
| 146 | u_result = main.ONOSbench.onosUninstall(main.ONOSip[i]) |
| 147 | utilities.assert_equals(expect=main.TRUE, actual=u_result, |
| 148 | onpass="Test step PASS", |
| 149 | onfail="Test step FAIL") |
| 150 | uninstallResult = (uninstallResult and u_result) |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 151 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 152 | main.step("Install ONOS package on all Nodes") |
| 153 | installResult = main.TRUE |
| 154 | for i in range(int(main.numCtrls)): |
| 155 | main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i]) |
| 156 | i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i]) |
| 157 | utilities.assert_equals(expect=main.TRUE, actual=i_result, |
| 158 | onpass="Test step PASS", |
| 159 | onfail="Test step FAIL") |
| 160 | installResult = installResult and i_result |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 161 | |
Chiyu Cheng | ef10950 | 2016-11-21 15:51:38 -0800 | [diff] [blame] | 162 | main.step( "Set up ONOS secure SSH" ) |
| 163 | secureSshResult = main.TRUE |
| 164 | for i in range( int( main.numCtrls ) ): |
| 165 | secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=main.ONOSip[i] ) |
| 166 | utilities.assert_equals( expect=main.TRUE, actual=secureSshResult, |
| 167 | onpass="Test step PASS", |
| 168 | onfail="Test step FAIL" ) |
| 169 | |
You Wang | 0357c43 | 2017-01-09 16:13:33 -0800 | [diff] [blame] | 170 | time.sleep( main.startUpSleep ) |
| 171 | main.step( "Starting ONOS service" ) |
| 172 | stopResult = main.TRUE |
| 173 | startResult = main.TRUE |
| 174 | onosIsUp = main.TRUE |
| 175 | for i in range( main.numCtrls ): |
| 176 | onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] ) |
| 177 | if onosIsUp == main.TRUE: |
| 178 | main.log.report( "ONOS instance is up and ready" ) |
| 179 | else: |
| 180 | main.log.report( "ONOS instance may not be up, stop and " + |
| 181 | "start ONOS again " ) |
| 182 | for i in range( main.numCtrls ): |
| 183 | stopResult = stopResult and \ |
| 184 | main.ONOSbench.onosStop( main.ONOSip[ i ] ) |
| 185 | for i in range( main.numCtrls ): |
| 186 | startResult = startResult and \ |
| 187 | main.ONOSbench.onosStart( main.ONOSip[ i ] ) |
| 188 | stepResult = onosIsUp and stopResult and startResult |
| 189 | utilities.assert_equals( expect=main.TRUE, |
| 190 | actual=stepResult, |
| 191 | onpass="ONOS service is ready", |
| 192 | onfail="ONOS service did not start properly" ) |
| 193 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 194 | time.sleep(2) |
| 195 | main.step("Start ONOS CLI on all nodes") |
| 196 | cliResult = main.TRUE |
Jon Hall | 6509dbf | 2016-06-21 17:01:17 -0700 | [diff] [blame] | 197 | main.step(" Start ONOS cli using thread ") |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 198 | startCliResult = main.TRUE |
| 199 | pool = [] |
| 200 | main.threadID = 0 |
| 201 | for i in range(int(main.numCtrls)): |
| 202 | t = main.Thread(target=main.CLIs[i].startOnosCli, |
| 203 | threadID=main.threadID, |
| 204 | name="startOnosCli", |
| 205 | args=[main.ONOSip[i]], |
| 206 | kwargs={"onosStartTimeout": main.timeout}) |
| 207 | pool.append(t) |
| 208 | t.start() |
| 209 | main.threadID = main.threadID + 1 |
| 210 | for t in pool: |
| 211 | t.join() |
| 212 | startCliResult = startCliResult and t.result |
| 213 | time.sleep(main.startUpSleep) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 214 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 215 | # configure apps |
| 216 | main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=7) |
| 217 | main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="linear") |
| 218 | main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true") |
YPZhang | 325822f | 2016-06-17 16:01:45 -0700 | [diff] [blame] | 219 | main.CLIs[0].setCfg("org.onosproject.net.intent.impl.IntentManager", "skipReleaseResourcesOnWithdrawal", value="true") |
YPZhang | e6ef82a | 2016-07-05 16:48:15 -0700 | [diff] [blame] | 220 | if main.flowObj: |
| 221 | main.CLIs[0].setCfg("org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator", |
| 222 | "useFlowObjectives", value="true") |
You Wang | 106d0fa | 2017-05-15 17:22:15 -0700 | [diff] [blame] | 223 | main.CLIs[0].setCfg("org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator", |
| 224 | "defaultFlowObjectiveCompiler", |
| 225 | value='org.onosproject.net.intent.impl.compiler.LinkCollectionIntentObjectiveCompiler') |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 226 | time.sleep(main.startUpSleep) |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 227 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 228 | # balanceMasters |
| 229 | main.CLIs[0].balanceMasters() |
You Wang | 3b148ea | 2016-06-20 20:33:42 -0700 | [diff] [blame] | 230 | time.sleep(main.startUpSleep) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 231 | |
| 232 | def CASE2( self, main ): |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 233 | import time |
| 234 | import numpy |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 235 | import json |
| 236 | print(main.intentsList) |
| 237 | for batchSize in main.intentsList: |
| 238 | main.log.report("Intent Batch size: {}".format(batchSize)) |
| 239 | main.installLatList = [] |
| 240 | main.withdrawLatList = [] |
| 241 | validrun = 0 |
| 242 | invalidrun = 0 |
| 243 | # we use two variables to control the iteration |
| 244 | while validrun <= main.warmUp + main.sampleSize and invalidrun < 20: |
| 245 | if validrun >= main.warmUp: |
| 246 | main.log.info("================================================") |
| 247 | main.log.info("Starting test iteration " + str(validrun - main.warmUp)) |
| 248 | main.log.info("Total test iteration: " + str(invalidrun + validrun)) |
| 249 | main.log.info("================================================") |
| 250 | else: |
| 251 | main.log.info("====================Warm Up=====================") |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 252 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 253 | # push intents |
| 254 | installResult = main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize, |
| 255 | offset=1, options="-i", timeout=main.timeout, |
| 256 | getResponse=True) |
| 257 | if type(installResult) is str: |
| 258 | if "Failure" in installResult: |
| 259 | main.log.error("Install Intents failure, ignore this iteration.") |
| 260 | if validrun < main.warmUp: |
| 261 | validrun += 1 |
| 262 | continue |
| 263 | else: |
| 264 | invalidrun += 1 |
| 265 | continue |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 266 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 267 | try: |
| 268 | latency = int(installResult.split()[5]) |
| 269 | main.log.info(installResult) |
| 270 | except: |
| 271 | main.log.error("Failed to get latency, ignore this iteration.") |
| 272 | main.log.error("Response from ONOS:") |
| 273 | print(installResult) |
| 274 | if validrun < main.warmUp: |
| 275 | validrun += 1 |
| 276 | continue |
| 277 | else: |
| 278 | invalidrun += 1 |
| 279 | continue |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 280 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 281 | if validrun >= main.warmUp: |
| 282 | main.installLatList.append(latency) |
| 283 | else: |
| 284 | invalidrun += 1 |
| 285 | continue |
| 286 | time.sleep(2) |
| 287 | # Withdraw Intents |
| 288 | withdrawResult = main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize, |
| 289 | offset=1, options="-w", timeout=main.timeout, |
| 290 | getResponse=True) |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 291 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 292 | if type(withdrawResult) is str: |
| 293 | if "Failure" in withdrawResult: |
| 294 | main.log.error("withdraw Intents failure, ignore this iteration.") |
| 295 | if validrun < main.warmUp: |
| 296 | validrun += 1 |
| 297 | continue |
| 298 | else: |
| 299 | invalidrun += 1 |
| 300 | continue |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 301 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 302 | try: |
| 303 | latency = int(withdrawResult.split()[5]) |
| 304 | main.log.info(withdrawResult) |
| 305 | except: |
| 306 | main.log.error("Failed to get latency, ignore this iteration.") |
| 307 | main.log.error("Response from ONOS:") |
| 308 | print(withdrawResult) |
| 309 | if validrun < main.warmUp: |
| 310 | validrun += 1 |
| 311 | continue |
| 312 | else: |
| 313 | invalidrun += 1 |
| 314 | continue |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 315 | |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 316 | if validrun >= main.warmUp: |
| 317 | main.withdrawLatList.append(latency) |
| 318 | else: |
| 319 | invalidrun += 1 |
| 320 | continue |
| 321 | time.sleep(2) |
| 322 | main.CLIs[0].purgeWithdrawnIntents() |
| 323 | validrun += 1 |
| 324 | installave = numpy.average(main.installLatList) |
| 325 | installstd = numpy.std(main.installLatList) |
| 326 | withdrawave = numpy.average(main.withdrawLatList) |
| 327 | withdrawstd = numpy.std(main.withdrawLatList) |
| 328 | # log report |
cameron@onlab.us | 41c16f5 | 2015-07-08 15:40:28 -0700 | [diff] [blame] | 329 | main.log.report("----------------------------------------------------") |
YPZhang | b34b7e1 | 2016-06-14 14:28:19 -0700 | [diff] [blame] | 330 | main.log.report("Scale: " + str(main.numCtrls)) |
| 331 | main.log.report("Intent batch: " + str(batchSize)) |
| 332 | main.log.report("Install average: {} std: {}".format(installave, installstd)) |
| 333 | main.log.report("Withdraw average: {} std: {}".format(withdrawave, withdrawstd)) |
| 334 | # write result to database file |
| 335 | if not (numpy.isnan(installave) or numpy.isnan(installstd) or\ |
| 336 | numpy.isnan(withdrawstd) or numpy.isnan(withdrawave)): |
| 337 | databaseString = "'" + main.commit + "'," |
| 338 | databaseString += str(main.numCtrls) + "," |
| 339 | databaseString += str(batchSize) + "," |
| 340 | databaseString += str(installave) + "," |
| 341 | databaseString += str(installstd) + "," |
| 342 | databaseString += str(withdrawave) + "," |
| 343 | databaseString += str(withdrawstd) + "\n" |
| 344 | resultsDB = open(main.dbFileName, "a") |
| 345 | resultsDB.write(databaseString) |
| 346 | resultsDB.close() |
| 347 | del main.scale[0] |