YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 1 | # SCPFintentRerouteLat |
| 2 | """ |
| 3 | SCPFintentRerouteLat |
| 4 | - Test Intent Reroute Latency |
| 5 | - Test Algorithm: |
| 6 | 1. Start Null Provider reroute Topology |
| 7 | 2. Using Push-test-intents to push batch size intents from switch 1 to switch 7 |
| 8 | 3. Cut the link between switch 3 and switch 4 (the path will reroute to switch 8) |
| 9 | 4. Get the topology time stamp |
| 10 | 5. Get Intent reroute(Installed) time stamp from each nodes |
| 11 | 6. Use the latest intent time stamp subtract topology time stamp |
| 12 | - This test will run 5 warm up by default, warm up iteration can be setup in Param file |
| 13 | - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file |
| 14 | - The unit of the latency result is milliseconds |
| 15 | """ |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 16 | |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 17 | |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 18 | class SCPFintentRerouteLat: |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 19 | def __init__(self): |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 20 | self.default = '' |
| 21 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 22 | def CASE0( self, main ): |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 23 | import imp |
| 24 | import os |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 25 | ''' |
| 26 | - GIT |
| 27 | - BUILDING ONOS |
| 28 | Pull specific ONOS branch, then Build ONOS ono ONOS Bench. |
| 29 | This step is usually skipped. Because in a Jenkins driven automated |
| 30 | test env. We want Jenkins jobs to pull&build for flexibility to handle |
| 31 | different versions of ONOS. |
| 32 | - Construct tests variables |
| 33 | ''' |
| 34 | gitPull = main.params['GIT']['gitPull'] |
| 35 | gitBranch = main.params['GIT']['gitBranch'] |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 36 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 37 | main.case("Pull onos branch and build onos on Teststation.") |
| 38 | |
| 39 | if gitPull == 'True': |
| 40 | main.step("Git Checkout ONOS branch: " + gitBranch) |
| 41 | stepResult = main.ONOSbench.gitCheckout(branch=gitBranch) |
| 42 | utilities.assert_equals(expect=main.TRUE, |
| 43 | actual=stepResult, |
| 44 | onpass="Successfully checkout onos branch.", |
| 45 | onfail="Failed to checkout onos branch. Exiting test...") |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 46 | if not stepResult: |
| 47 | main.exit() |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 48 | |
| 49 | main.step("Git Pull on ONOS branch:" + gitBranch) |
| 50 | stepResult = main.ONOSbench.gitPull() |
| 51 | utilities.assert_equals(expect=main.TRUE, |
| 52 | actual=stepResult, |
| 53 | onpass="Successfully pull onos. ", |
| 54 | onfail="Failed to pull onos. Exiting test ...") |
| 55 | if not stepResult: main.exit() |
| 56 | |
| 57 | main.step("Building ONOS branch: " + gitBranch) |
| 58 | stepResult = main.ONOSbench.cleanInstall(skipTest=True) |
| 59 | utilities.assert_equals(expect=main.TRUE, |
| 60 | actual=stepResult, |
| 61 | onpass="Successfully build onos.", |
| 62 | onfail="Failed to build onos. Exiting test...") |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 63 | if not stepResult: |
| 64 | main.exit() |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 65 | |
| 66 | else: |
| 67 | main.log.warn("Skipped pulling onos and Skipped building ONOS") |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 68 | main.onosIp = main.ONOSbench.getOnosIps() |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 69 | main.apps = main.params['ENV']['cellApps'] |
| 70 | main.BENCHUser = main.params['BENCH']['user'] |
| 71 | main.BENCHIp = main.params['BENCH']['ip1'] |
| 72 | main.MN1Ip = main.params['MN']['ip1'] |
| 73 | main.maxNodes = int(main.params['max']) |
| 74 | main.skipMvn = main.params['TEST']['skipCleanInstall'] |
| 75 | main.cellName = main.params['ENV']['cellName'] |
| 76 | main.scale = (main.params['SCALE']).split(",") |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 77 | main.timeout = int(main.params['SLEEP']['timeout']) |
| 78 | main.startUpSleep = int(main.params['SLEEP']['startup']) |
| 79 | main.installSleep = int(main.params['SLEEP']['install']) |
| 80 | main.verifySleep = int(main.params['SLEEP']['verify']) |
YPZhang | 2b9b26d | 2016-06-20 16:18:29 -0700 | [diff] [blame] | 81 | main.setMasterSleep = int(main.params['SLEEP']['setmaster']) |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 82 | main.verifyAttempts = int(main.params['ATTEMPTS']['verify']) |
You Wang | 0b9039d | 2017-01-12 16:51:29 -0800 | [diff] [blame] | 83 | main.maxInvalidRun = int(main.params['ATTEMPTS']['maxInvalidRun']) |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 84 | main.sampleSize = int(main.params['TEST']['sampleSize']) |
| 85 | main.warmUp = int(main.params['TEST']['warmUp']) |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 86 | main.ingress = main.params['TEST']['ingress'] |
| 87 | main.egress = main.params['TEST']['egress'] |
| 88 | main.debug = main.params['TEST']['debug'] |
YPZhang | e6ef82a | 2016-07-05 16:48:15 -0700 | [diff] [blame] | 89 | main.flowObj = main.params['TEST']['flowObj'] |
Chiyu Cheng | 1976db5 | 2016-10-21 15:57:30 -0700 | [diff] [blame] | 90 | main.deviceCount = int(main.params['TEST']['deviceCount']) |
| 91 | main.end1 = main.params['TEST']['end1'] |
| 92 | main.end2 = main.params['TEST']['end2'] |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 93 | main.searchTerm = main.params['SEARCHTERM'] |
YPZhang | e6ef82a | 2016-07-05 16:48:15 -0700 | [diff] [blame] | 94 | if main.flowObj == "True": |
| 95 | main.flowObj = True |
| 96 | main.dbFileName = main.params['DATABASE']['dbFlowObj'] |
YPZhang | 3b5a78f | 2016-07-14 10:43:33 -0700 | [diff] [blame] | 97 | main.intentsList = (main.params['TEST']['FObjintents']).split(",") |
YPZhang | e6ef82a | 2016-07-05 16:48:15 -0700 | [diff] [blame] | 98 | else: |
| 99 | main.flowObj = False |
| 100 | main.dbFileName = main.params['DATABASE']['dbName'] |
YPZhang | 3b5a78f | 2016-07-14 10:43:33 -0700 | [diff] [blame] | 101 | main.intentsList = (main.params['TEST']['intents']).split(",") |
YPZhang | e6ef82a | 2016-07-05 16:48:15 -0700 | [diff] [blame] | 102 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 103 | for i in range(0, len(main.intentsList)): |
| 104 | main.intentsList[i] = int(main.intentsList[i]) |
| 105 | # Create DataBase file |
| 106 | main.log.info("Create Database file " + main.dbFileName) |
| 107 | resultsDB = open(main.dbFileName, "w+") |
| 108 | resultsDB.close() |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 109 | file1 = main.params[ "DEPENDENCY" ][ "FILE1" ] |
| 110 | main.dependencyPath = os.path.dirname( os.getcwd() ) + main.params[ "DEPENDENCY" ][ "PATH" ] |
| 111 | main.intentRerouteLatFuncs = imp.load_source(file1, main.dependencyPath + file1 + ".py") |
| 112 | |
| 113 | main.record = 0 |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 114 | |
| 115 | def CASE1( self, main ): |
| 116 | ''' |
| 117 | clean up test environment and set up |
| 118 | ''' |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 119 | import time |
| 120 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 121 | main.log.info("Get ONOS cluster IP") |
| 122 | print(main.scale) |
| 123 | main.numCtrls = int(main.scale[0]) |
| 124 | main.ONOSip = [] |
| 125 | main.maxNumBatch = 0 |
| 126 | main.AllONOSip = main.ONOSbench.getOnosIps() |
| 127 | for i in range(main.numCtrls): |
| 128 | main.ONOSip.append(main.AllONOSip[i]) |
| 129 | main.log.info(main.ONOSip) |
| 130 | main.CLIs = [] |
| 131 | main.log.info("Creating list of ONOS cli handles") |
| 132 | for i in range(main.numCtrls): |
| 133 | main.CLIs.append(getattr(main, 'ONOS%scli' % (i + 1))) |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 134 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 135 | if not main.CLIs: |
| 136 | main.log.error("Failed to create the list of ONOS cli handles") |
| 137 | main.cleanup() |
| 138 | main.exit() |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 139 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 140 | main.commit = main.ONOSbench.getVersion(report=True) |
| 141 | main.commit = main.commit.split(" ")[1] |
| 142 | main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls) |
| 143 | main.log.info("Safety check, killing all ONOS processes" + |
| 144 | " before initiating environment setup") |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 145 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 146 | for i in range(main.numCtrls): |
You Wang | b98a9fa | 2017-02-15 17:27:42 -0800 | [diff] [blame] | 147 | main.ONOSbench.onosStop(main.ONOSip[i]) |
| 148 | main.ONOSbench.onosKill(main.ONOSip[i]) |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 149 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 150 | main.log.info("NODE COUNT = %s" % main.numCtrls) |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 151 | main.ONOSbench.createCellFile(main.ONOSbench.ip_address, |
| 152 | main.cellName, |
| 153 | main.MN1Ip, |
| 154 | main.apps, |
| 155 | main.ONOSip) |
| 156 | main.step("Apply cell to environment") |
| 157 | cellResult = main.ONOSbench.setCell(main.cellName) |
| 158 | verifyResult = main.ONOSbench.verifyCell() |
| 159 | stepResult = cellResult and verifyResult |
| 160 | utilities.assert_equals(expect=main.TRUE, |
| 161 | actual=stepResult, |
| 162 | onpass="Successfully applied cell to " + \ |
| 163 | "environment", |
| 164 | onfail="Failed to apply cell to environment ") |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 165 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 166 | main.step("Creating ONOS package") |
Jon Hall | bd60ea0 | 2016-08-23 10:03:59 -0700 | [diff] [blame] | 167 | packageResult = main.ONOSbench.buckBuild() |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 168 | stepResult = packageResult |
| 169 | utilities.assert_equals(expect=main.TRUE, |
| 170 | actual=stepResult, |
| 171 | onpass="Successfully created ONOS package", |
| 172 | onfail="Failed to create ONOS package") |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 173 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 174 | main.step("Uninstall ONOS package on all Nodes") |
| 175 | uninstallResult = main.TRUE |
| 176 | for i in range(int(main.numCtrls)): |
| 177 | main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i]) |
| 178 | u_result = main.ONOSbench.onosUninstall(main.ONOSip[i]) |
| 179 | utilities.assert_equals(expect=main.TRUE, actual=u_result, |
| 180 | onpass="Test step PASS", |
| 181 | onfail="Test step FAIL") |
| 182 | uninstallResult = (uninstallResult and u_result) |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 183 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 184 | main.step("Install ONOS package on all Nodes") |
| 185 | installResult = main.TRUE |
| 186 | for i in range(int(main.numCtrls)): |
| 187 | main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i]) |
| 188 | i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i]) |
| 189 | utilities.assert_equals(expect=main.TRUE, actual=i_result, |
| 190 | onpass="Test step PASS", |
| 191 | onfail="Test step FAIL") |
| 192 | installResult = installResult and i_result |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 193 | |
You Wang | f5de25b | 2017-01-06 15:13:01 -0800 | [diff] [blame] | 194 | main.step( "Set up ONOS secure SSH" ) |
| 195 | secureSshResult = main.TRUE |
| 196 | for i in range( int( main.numCtrls ) ): |
| 197 | secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=main.ONOSip[i] ) |
| 198 | utilities.assert_equals( expect=main.TRUE, actual=secureSshResult, |
| 199 | onpass="Test step PASS", |
| 200 | onfail="Test step FAIL" ) |
| 201 | |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 202 | main.step( "Starting ONOS service" ) |
| 203 | stopResult = main.TRUE |
| 204 | startResult = main.TRUE |
| 205 | onosIsUp = main.TRUE |
Chiyu Cheng | ef10950 | 2016-11-21 15:51:38 -0800 | [diff] [blame] | 206 | |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 207 | for i in range( main.numCtrls ): |
| 208 | onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] ) |
| 209 | if onosIsUp == main.TRUE: |
| 210 | main.log.report( "ONOS instance is up and ready" ) |
| 211 | else: |
| 212 | main.log.report( "ONOS instance may not be up, stop and " + |
| 213 | "start ONOS again " ) |
| 214 | |
| 215 | for i in range( main.numCtrls ): |
| 216 | stopResult = stopResult and \ |
| 217 | main.ONOSbench.onosStop( main.ONOSip[ i ] ) |
| 218 | for i in range( main.numCtrls ): |
| 219 | startResult = startResult and \ |
| 220 | main.ONOSbench.onosStart( main.ONOSip[ i ] ) |
| 221 | stepResult = onosIsUp and stopResult and startResult |
| 222 | utilities.assert_equals( expect=main.TRUE, actual=stepResult, |
| 223 | onpass="Test step PASS", |
| 224 | onfail="Test step FAIL" ) |
You Wang | f5de25b | 2017-01-06 15:13:01 -0800 | [diff] [blame] | 225 | |
You Wang | 0b9039d | 2017-01-12 16:51:29 -0800 | [diff] [blame] | 226 | time.sleep(main.startUpSleep) |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 227 | main.step("Start ONOS CLI on all nodes") |
| 228 | cliResult = main.TRUE |
Jon Hall | 6509dbf | 2016-06-21 17:01:17 -0700 | [diff] [blame] | 229 | main.step(" Start ONOS cli using thread ") |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 230 | startCliResult = main.TRUE |
| 231 | pool = [] |
| 232 | main.threadID = 0 |
| 233 | for i in range(int(main.numCtrls)): |
| 234 | t = main.Thread(target=main.CLIs[i].startOnosCli, |
| 235 | threadID=main.threadID, |
| 236 | name="startOnosCli", |
| 237 | args=[main.ONOSip[i]], |
| 238 | kwargs={"onosStartTimeout": main.timeout}) |
| 239 | pool.append(t) |
| 240 | t.start() |
| 241 | main.threadID = main.threadID + 1 |
| 242 | for t in pool: |
| 243 | t.join() |
| 244 | startCliResult = startCliResult and t.result |
| 245 | time.sleep(main.startUpSleep) |
Jon Hall | 4ba53f0 | 2015-07-29 13:07:41 -0700 | [diff] [blame] | 246 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 247 | # configure apps |
Chiyu Cheng | 1976db5 | 2016-10-21 15:57:30 -0700 | [diff] [blame] | 248 | main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=main.deviceCount) |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 249 | main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="reroute") |
| 250 | main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true") |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 251 | # "bakcupEnabled" should be true by default. Not sure if it's intended or not to disable it. Seems no impact to test results. |
| 252 | #main.CLIs[0].setCfg("org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled", value="false") |
YPZhang | e6ef82a | 2016-07-05 16:48:15 -0700 | [diff] [blame] | 253 | if main.flowObj: |
| 254 | main.CLIs[0].setCfg("org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator", |
| 255 | "useFlowObjectives", value="true") |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 256 | time.sleep( main.startUpSleep ) |
| 257 | for i in range( int( main.numCtrls ) ): |
| 258 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.metrics.topology" ) |
| 259 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.metrics.intent" ) |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 260 | # Balance Master |
| 261 | main.CLIs[0].balanceMasters() |
You Wang | 0b9039d | 2017-01-12 16:51:29 -0800 | [diff] [blame] | 262 | time.sleep( main.setMasterSleep ) |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 263 | if len(main.ONOSip) > 1: |
Chiyu Cheng | 1976db5 | 2016-10-21 15:57:30 -0700 | [diff] [blame] | 264 | main.CLIs[0].deviceRole(main.end1[ 'name' ], main.ONOSip[0]) |
| 265 | main.CLIs[0].deviceRole(main.end2[ 'name' ], main.ONOSip[0]) |
YPZhang | 2b9b26d | 2016-06-20 16:18:29 -0700 | [diff] [blame] | 266 | time.sleep( main.setMasterSleep ) |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 267 | |
| 268 | def CASE2( self, main ): |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 269 | import time |
| 270 | import numpy |
| 271 | import datetime |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 272 | import json |
| 273 | # from scipy import stats |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 274 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 275 | print(main.intentsList) |
| 276 | for batchSize in main.intentsList: |
You Wang | 0b9039d | 2017-01-12 16:51:29 -0800 | [diff] [blame] | 277 | main.batchSize = batchSize |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 278 | main.log.report("Intent Batch size: " + str(batchSize) + "\n ") |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 279 | firstLocalLatencies = [] |
| 280 | lastLocalLatencies = [] |
| 281 | firstGlobalLatencies = [] |
| 282 | lastGlobalLatencies = [] |
| 283 | main.startLine = {} |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 284 | main.validRun = 0 |
| 285 | main.invalidRun = 0 |
You Wang | 0b9039d | 2017-01-12 16:51:29 -0800 | [diff] [blame] | 286 | while main.validRun <= main.warmUp + main.sampleSize and main.invalidRun <= main.maxInvalidRun: |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 287 | if main.validRun >= main.warmUp: |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 288 | main.log.info("================================================") |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 289 | main.log.info("Valid iteration: {} ".format( main.validRun - main.warmUp)) |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 290 | main.log.info("Total iteration: {}".format( main.validRun + main.invalidRun)) |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 291 | main.log.info("================================================") |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 292 | else: |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 293 | main.log.info("====================Warm Up=====================") |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 294 | |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 295 | # push intents |
You Wang | 0b9039d | 2017-01-12 16:51:29 -0800 | [diff] [blame] | 296 | main.CLIs[0].pushTestIntents(main.ingress, main.egress, main.batchSize, |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 297 | offset=1, options="-i", timeout=main.timeout) |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 298 | |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 299 | # check links, flows and intents |
| 300 | main.intentRerouteLatFuncs.sanityCheck( main, main.deviceCount * 2, batchSize * (main.deviceCount - 1 ), main.batchSize ) |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 301 | if not main.verify: |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 302 | main.log.warn( "Sanity check failed, skipping this iteration..." ) |
| 303 | continue |
| 304 | |
| 305 | # Insert one line in karaf.log before link down |
| 306 | for i in range( main.numCtrls ): |
| 307 | main.CLIs[ i ].log( "\'Scale: {}, Batch:{}, Iteration: {}\'".format( main.numCtrls, batchSize, main.validRun + main.invalidRun ) ) |
| 308 | |
| 309 | # bring link down |
Chiyu Cheng | 1976db5 | 2016-10-21 15:57:30 -0700 | [diff] [blame] | 310 | main.CLIs[0].link( main.end1[ 'port' ], main.end2[ 'port' ], "down", |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 311 | timeout=main.timeout, showResponse=False) |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 312 | |
| 313 | # check links, flows and intents |
| 314 | main.intentRerouteLatFuncs.sanityCheck( main, (main.deviceCount - 1) * 2, batchSize * main.deviceCount, main.batchSize ) |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 315 | if not main.verify: |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 316 | main.log.warn( "Sanity check failed, skipping this iteration..." ) |
| 317 | continue |
| 318 | |
| 319 | # Get timestamp of last LINK_REMOVED event as separator between iterations |
| 320 | skip = False |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 321 | for i in range( main.numCtrls ): |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 322 | logNum = main.intentRerouteLatFuncs.getLogNum( main, i ) |
| 323 | timestamp = str( main.CLIs[ i ].getTimeStampFromLog( "last", "LINK_REMOVED", "time = ", " ", logNum=logNum ) ) |
| 324 | if timestamp == main.ERROR: |
| 325 | # Try again in case that the log number just increased |
| 326 | logNum = main.intentRerouteLatFuncs.getLogNum( main, i ) |
| 327 | timestamp = str( main.CLIs[ i ].getTimeStampFromLog( "last", "LINK_REMOVED", "time = ", " ", logNum=logNum ) ) |
| 328 | if timestamp == main.ERROR: |
| 329 | main.log.warn( "Cannot find the event we want in the log, skipping this iteration..." ) |
| 330 | main.intentRerouteLatFuncs.bringBackTopology( main ) |
| 331 | if main.validRun >= main.warmUp: |
| 332 | main.invalidRun += 1 |
| 333 | else: |
| 334 | main.validRun += 1 |
| 335 | skip = True |
| 336 | break |
| 337 | else: |
| 338 | main.startLine[ i ] = timestamp |
| 339 | main.log.info( "Timestamp of last LINK_REMOVED event on node {} is {}".format( i+1, main.startLine[ i ] ) ) |
| 340 | if skip: continue |
| 341 | |
| 342 | # calculate values |
| 343 | topologyTimestamps = main.intentRerouteLatFuncs.getTopologyTimestamps( main ) |
| 344 | intentTimestamps = main.intentRerouteLatFuncs.getIntentTimestamps( main ) |
| 345 | if intentTimestamps == main.ERROR or topologyTimestamps == main.ERROR: |
| 346 | main.log.info( "Got invalid timestamp, skipping this iteration..." ) |
| 347 | main.intentRerouteLatFuncs.bringBackTopology( main ) |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 348 | if main.validRun >= main.warmUp: |
| 349 | main.invalidRun += 1 |
YPZhang | 8742d2e | 2016-06-16 15:31:58 -0700 | [diff] [blame] | 350 | else: |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 351 | main.validRun += 1 |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 352 | continue |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 353 | else: |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 354 | main.log.info( "Got valid timestamps" ) |
| 355 | |
| 356 | firstLocalLatnecy, lastLocalLatnecy, firstGlobalLatency, lastGlobalLatnecy = main.intentRerouteLatFuncs.calculateLatency( main, topologyTimestamps, intentTimestamps ) |
| 357 | if firstLocalLatnecy < 0: |
| 358 | main.log.info( "Got negative latency, skipping this iteration..." ) |
| 359 | main.intentRerouteLatFuncs.bringBackTopology( main ) |
| 360 | if main.validRun >= main.warmUp: |
| 361 | main.invalidRun += 1 |
| 362 | else: |
| 363 | main.validRun += 1 |
| 364 | continue |
| 365 | else: |
| 366 | main.log.info( "Got valid latencies" ) |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 367 | main.validRun += 1 |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 368 | |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 369 | firstLocalLatencies.append( firstLocalLatnecy ) |
| 370 | lastLocalLatencies.append( lastLocalLatnecy ) |
| 371 | firstGlobalLatencies.append( firstGlobalLatency ) |
| 372 | lastGlobalLatencies.append( lastGlobalLatnecy ) |
| 373 | |
| 374 | # bring up link and withdraw intents |
Chiyu Cheng | 1976db5 | 2016-10-21 15:57:30 -0700 | [diff] [blame] | 375 | main.CLIs[0].link( main.end1[ 'port' ], main.end2[ 'port' ], "up", |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 376 | timeout=main.timeout) |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 377 | main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize, |
| 378 | offset=1, options="-w", timeout=main.timeout) |
| 379 | main.CLIs[0].purgeWithdrawnIntents() |
| 380 | |
| 381 | # check links, flows and intents |
| 382 | main.intentRerouteLatFuncs.sanityCheck( main, main.deviceCount * 2, 0, 0 ) |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 383 | if not main.verify: |
YPZhang | febf730 | 2016-05-24 16:45:56 -0700 | [diff] [blame] | 384 | continue |
| 385 | |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 386 | aveLocalLatency = numpy.average( lastLocalLatencies ) |
| 387 | aveGlobalLatency = numpy.average( lastGlobalLatencies ) |
| 388 | stdLocalLatency = numpy.std( lastLocalLatencies ) |
| 389 | stdGlobalLatency = numpy.std( lastGlobalLatencies ) |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 390 | |
| 391 | main.log.report( "Scale: " + str( main.numCtrls ) + " \tIntent batch: " + str( batchSize ) ) |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 392 | main.log.report( "Local latency average:................" + str( aveLocalLatency ) ) |
| 393 | main.log.report( "Global latency average:................" + str( aveGlobalLatency ) ) |
| 394 | main.log.report( "Local latency std:................" + str( stdLocalLatency ) ) |
| 395 | main.log.report( "Global latency std:................" + str( stdGlobalLatency ) ) |
Chiyu Cheng | ec63bde | 2016-11-17 18:11:36 -0800 | [diff] [blame] | 396 | main.log.report( "________________________________________________________" ) |
cameron@onlab.us | 78b8965 | 2015-07-08 15:21:03 -0700 | [diff] [blame] | 397 | |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 398 | if not ( numpy.isnan( aveLocalLatency ) or numpy.isnan( aveGlobalLatency ) ): |
YPZhang | 8742d2e | 2016-06-16 15:31:58 -0700 | [diff] [blame] | 399 | # check if got NaN for result |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 400 | resultsDB = open( main.dbFileName, "a" ) |
| 401 | resultsDB.write( "'" + main.commit + "'," ) |
| 402 | resultsDB.write( str( main.numCtrls ) + "," ) |
| 403 | resultsDB.write( str( batchSize ) + "," ) |
| 404 | resultsDB.write( str( aveLocalLatency ) + "," ) |
| 405 | resultsDB.write( str( stdLocalLatency ) + "\n" ) |
YPZhang | 8742d2e | 2016-06-16 15:31:58 -0700 | [diff] [blame] | 406 | resultsDB.close() |
You Wang | 6d301d4 | 2017-04-21 10:49:33 -0700 | [diff] [blame] | 407 | del main.scale[ 0 ] |