Package TestON :: Package tests :: Package SCPFintentRerouteLat :: Module SCPFintentRerouteLat
[hide private]
[frames] | no frames]

Source Code for Module TestON.tests.SCPFintentRerouteLat.SCPFintentRerouteLat

  1  # ScaleOutTemplate 
  2  # 
  3  # CASE1 starts number of nodes specified in param file 
  4  # 
  5  # cameron@onlab.us 
  6   
  7  import sys 
  8  import os.path 
  9   
 10   
11 -class SCPFintentRerouteLat:
12
13 - def __init__( self ):
14 self.default = ''
15
16 - def CASE1( self, main ):
17 18 import time 19 20 global init 21 try: 22 if type(init) is not bool: 23 init = False 24 except NameError: 25 init = False 26 27 #Load values from params file 28 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ] 29 gitPull = main.params[ 'GIT' ][ 'autopull' ] 30 cellName = main.params[ 'ENV' ][ 'cellName' ] 31 Apps = main.params[ 'ENV' ][ 'cellApps' ] 32 BENCHUser = main.params[ 'BENCH' ][ 'user' ] 33 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ] 34 MN1Ip = main.params[ 'MN' ][ 'ip1' ] 35 main.maxNodes = int(main.params[ 'max' ]) 36 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ] 37 cellName = main.params[ 'ENV' ][ 'cellName' ] 38 39 # -- INIT SECTION, ONLY RUNS ONCE -- # 40 if init == False: 41 init = True 42 global clusterCount #number of nodes running 43 global ONOSIp #list of ONOS IP addresses 44 global scale 45 global commit 46 47 clusterCount = 0 48 ONOSIp = [ 0 ] 49 scale = (main.params[ 'SCALE' ]).split(",") 50 clusterCount = int(scale[0]) 51 52 #Populate ONOSIp with ips from params 53 ONOSIp = [0] 54 ONOSIp.extend(main.ONOSbench.getOnosIps()) 55 56 print("-----------------" + str(ONOSIp)) 57 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test 58 if skipMvn != "yes": 59 mvnResult = main.ONOSbench.cleanInstall() 60 61 #git 62 main.step( "Git checkout and pull " + checkoutBranch ) 63 if gitPull == 'on': 64 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch ) 65 pullResult = main.ONOSbench.gitPull() 66 67 else: 68 checkoutResult = main.TRUE 69 pullResult = main.TRUE 70 main.log.info( "Skipped git checkout and pull" ) 71 72 commit = main.ONOSbench.getVersion() 73 commit = (commit.split(" "))[1] 74 75 resultsDB = open("/tmp/IntentRerouteLatDB", "w+") 76 resultsDB.close() 77 78 # -- END OF INIT SECTION --# 79 80 clusterCount = int(scale[0]) 81 scale.remove(scale[0]) 82 83 #kill off all onos processes 84 main.log.step("Safety check, killing all ONOS processes") 85 main.log.step("before initiating enviornment setup") 86 for node in range(1, main.maxNodes + 1): 87 main.ONOSbench.onosDie(ONOSIp[node]) 88 89 #Uninstall everywhere 90 main.log.step( "Cleaning Enviornment..." ) 91 for i in range(1, main.maxNodes + 1): 92 main.log.info(" Uninstalling ONOS " + str(i) ) 93 main.ONOSbench.onosUninstall( ONOSIp[i] ) 94 95 #construct the cell file 96 main.log.info("Creating cell file") 97 cellIp = [] 98 for node in range (1, clusterCount + 1): 99 cellIp.append(ONOSIp[node]) 100 101 main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), cellIp) 102 103 main.step( "Set Cell" ) 104 main.ONOSbench.setCell(cellName) 105 106 main.step( "Creating ONOS package" ) 107 packageResult = main.ONOSbench.onosPackage() 108 109 main.step( "verify cells" ) 110 verifyCellResult = main.ONOSbench.verifyCell() 111 112 main.log.report( "Initializing " + str( clusterCount ) + " node cluster." ) 113 for node in range(1, clusterCount + 1): 114 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node]) 115 main.ONOSbench.onosInstall( ONOSIp[node]) 116 117 for node in range(1, clusterCount + 1): 118 for i in range( 2 ): 119 isup = main.ONOSbench.isup( ONOSIp[node] ) 120 if isup: 121 main.log.info("ONOS " + str(node) + " is up\n") 122 break 123 if not isup: 124 main.log.report( "ONOS " + str(node) + " didn't start!" ) 125 main.log.info("Startup sequence complete") 126 127 deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",") 128 print("Device mastership list: " + str(deviceMastership)) 129 130 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.store.flow.impl.NewDistributedFlowRuleStore", "backupEnabled false") 131 132 main.log.step("Setting up null provider") 133 for i in range(3): 134 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "deviceCount 8") 135 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "topoShape reroute") 136 main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "enabled true") 137 time.sleep(5) 138 main.ONOSbench.handle.sendline("onos $OC1 summary") 139 main.ONOSbench.handle.expect(":~") 140 x = main.ONOSbench.handle.before 141 if "devices=8" in x and "links=16," in x: 142 break 143 144 index = 1 145 for node in deviceMastership: 146 for attempt in range(0,10): 147 cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)] + """ master" """) 148 main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n") 149 main.ONOSbench.handle.sendline(cmd) 150 main.ONOSbench.handle.expect(":~") 151 time.sleep(4) 152 153 cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index)) 154 main.log.info(cmd) 155 main.ONOSbench.handle.sendline(cmd) 156 main.ONOSbench.handle.expect(":~") 157 check = main.ONOSbench.handle.before 158 main.log.info("CHECK:\n" + check) 159 if ("master=" + ONOSIp[int(node)]) in check: 160 break 161 index += 1 162 163 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
164
165 - def CASE2( self, main ):
166 167 import time 168 import numpy 169 import datetime 170 #from scipy import stats 171 172 ts = time.time() 173 date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d') 174 175 sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ]) 176 warmUp = int(main.params[ 'TEST' ][ 'warmUp' ]) 177 intentsList = (main.params[ 'TEST' ][ 'intents' ]).split(",") 178 debug = main.params[ 'TEST' ][ 'debug' ] 179 for i in range(0,len(intentsList)): 180 intentsList[i] = int(intentsList[i]) 181 182 timestampMetrics = [] 183 if main.params['METRICS']['Submitted'] == "1": 184 timestampMetrics.append("Submitted") 185 if main.params['METRICS']['Installed'] == "1": 186 timestampMetrics.append("Installed") 187 if main.params['METRICS']['Failed'] == "1": 188 timestampMetrics.append("Failed") 189 if main.params['METRICS']['Withdraw'] == "1": 190 timestampMetrics.append("Withdraw") 191 if main.params['METRICS']['Withdrawn'] == "1": 192 timestampMetrics.append("Withdrawn") 193 if debug: main.log.info(timestampMetrics) 194 195 if debug == "True": 196 debug = True 197 else: 198 debug = False 199 200 ingress = "null:0000000000000001" 201 egress = "null:0000000000000007" 202 203 for intents in intentsList: 204 main.log.report("Intent Batch size: " + str(intents) + "\n ") 205 myResult = [["latency", "lastNode"] for x in range(sampleSize)] 206 207 for run in range(0, (warmUp + sampleSize)): 208 if run > warmUp: 209 main.log.info("Starting test iteration " + str(run-warmUp)) 210 211 cmd = """onos $OC1 "push-test-intents -i """ 212 cmd += ingress + "/0 " 213 cmd += egress + "/0 " 214 cmd += str(intents) +""" 1" """ 215 if debug: main.log.info(cmd) 216 217 withdrawCmd = cmd.replace("intents -i", "intents -w ") 218 219 #push-test-intents 220 main.ONOSbench.handle.sendline(cmd) 221 main.ONOSbench.handle.expect(":~") 222 myRawResult = main.ONOSbench.handle.before 223 224 for i in range(0, 40): 225 main.ONOSbench.handle.sendline("onos $OC1 summary") 226 main.ONOSbench.handle.expect(":~") 227 linkCheck = main.ONOSbench.handle.before 228 if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","): 229 break 230 if i == 39: 231 main.log.error("Flow/link count incorrect, data invalid."+ linkCheck) 232 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d") 233 #main.ONOSbench.logReport(ONOSIp[(clusterCount-1)], ["ERROR", "WARNING", "EXCEPT"], "d") 234 main.ONOSbench.sendline("onos $OC1 summary") 235 main.ONOSbench.sendline("onos $OC1 devices") 236 main.ONOSbench.sendline("onos $OC1 links") 237 main.ONOSbench.expect(":~") 238 main.log.info(main.ONOSbench.before) 239 240 #collect timestamp from link cut 241 cmd = """onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 down" """ 242 if debug: main.log.info("COMMAND: " + str(cmd)) 243 main.ONOSbench.handle.sendline(cmd) 244 main.ONOSbench.handle.expect(":~") 245 246 cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1" 247 for i in range(0,10): 248 main.ONOSbench.handle.sendline(cmd) 249 time.sleep(2) 250 main.ONOSbench.handle.expect(":~") 251 raw = main.ONOSbench.handle.before 252 #if "NullLinkProvider" in raw and "links=14" in raw: 253 if "links=14" in raw: 254 break 255 if i >= 9: 256 main.log.error("Expected output not being recieved... continuing") 257 main.log.info(raw) 258 break 259 time.sleep(2) 260 261 temp = raw.splitlines() 262 for line in temp: 263 if str(date) in line: 264 temp = line 265 break 266 267 cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1] 268 if debug: main.log.info("Cut timestamp: " + cutTimestamp) 269 270 #validate link count and flow count 271 for i in range(0, 40): 272 main.ONOSbench.handle.sendline("onos $OC1 summary") 273 main.ONOSbench.handle.expect(":~") 274 linkCheck = main.ONOSbench.handle.before 275 #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck: 276 if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck: 277 break 278 if i == 39: 279 main.log.error("Link or flow count incorrect, data invalid." + linkCheck) 280 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d") 281 282 time.sleep(5) #trying to avoid negative values 283 284 #intents events metrics installed timestamp 285 IEMtimestamps = [0]*(clusterCount + 1) 286 installedTemp = [0]*(clusterCount + 1) 287 for node in range(1, clusterCount +1): 288 cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """ 289 raw = "" 290 while "epoch)" not in raw: 291 main.ONOSbench.handle.sendline(cmd) 292 main.ONOSbench.handle.expect(":~") 293 raw = main.ONOSbench.handle.before 294 295 print(raw) 296 297 intentsTimestamps = {} 298 rawTimestamps = raw.splitlines() 299 for line in rawTimestamps: 300 if "Timestamp" in line and "grep" not in line: 301 metricKey = (line.split(" "))[1] 302 metricTimestamp = (line.split(" ")[len(line.split(" ")) -1]).replace("epoch)=","") 303 metricTimestamp = float(metricTimestamp) 304 metricTimestamp = numpy.divide(metricTimestamp, 1000) 305 if debug: main.log.info(repr(metricTimestamp)) 306 intentsTimestamps[metricKey] = metricTimestamp 307 if metricKey == "Installed": 308 installedTemp[node] = metricTimestamp 309 310 main.log.info("Node: " + str(node) + " Timestamps: " + str(intentsTimestamps)) 311 IEMtimestamps[node] = intentsTimestamps 312 313 myMax = max(installedTemp) 314 indexOfMax = installedTemp.index(myMax) 315 316 #number crunch 317 for metric in timestampMetrics: #this is where we sould add support for computing other timestamp metrics 318 if metric == "Installed": 319 if run >= warmUp: 320 main.log.report("link cut timestamp: " + cutTimestamp) 321 #readableInstalledTimestamp = str(intentsTimestamps["Installed"]) 322 readableInstalledTimestamp = str(myMax) 323 324 #main.log.report("Intent Installed timestamp: " + str(intentsTimestamps["Installed"])) 325 main.log.report("Intent Installed timestamp: " + str(myMax)) 326 327 cutEpoch = time.mktime(time.strptime(cutTimestamp, "%Y-%m-%d %H:%M:%S,%f")) 328 if debug: main.log.info("cutEpoch=" + str(cutEpoch)) 329 #rerouteLatency = float(intentsTimestamps["Installed"] - cutEpoch) 330 rerouteLatency = float(myMax - cutEpoch) 331 332 rerouteLatency = numpy.divide(rerouteLatency, 1000) 333 main.log.report("Reroute latency:" + str(rerouteLatency) + " (seconds)\n ") 334 myResult[run-warmUp][0] = rerouteLatency 335 myResult[run-warmUp][1] = indexOfMax 336 if debug: main.log.info("Latency: " + str(myResult[run-warmUp][0])) 337 if debug: main.log.info("last node: " + str(myResult[run-warmUp][1])) 338 339 cmd = """ onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 up" """ 340 if debug: main.log.info(cmd) 341 main.ONOSbench.handle.sendline(cmd) 342 main.ONOSbench.handle.expect(":~") 343 344 #wait for intent withdraw 345 main.ONOSbench.handle.sendline(withdrawCmd) 346 main.log.info(withdrawCmd) 347 main.ONOSbench.handle.expect(":~") 348 if debug: main.log.info(main.ONOSbench.handle.before) 349 main.ONOSbench.handle.sendline("onos $OC1 intents|grep WITHDRAWN|wc -l") 350 main.ONOSbench.handle.expect(":~") 351 intentWithdrawCheck = main.ONOSbench.handle.before 352 if (str(intents)) in intentWithdrawCheck: 353 main.log.info("intents withdrawn") 354 if debug: main.log.info(intentWithdrawCheck) 355 356 # wait for links to be reestablished 357 for i in range(0, 10): 358 main.ONOSbench.handle.sendline("onos $OC1 summary") 359 main.ONOSbench.handle.expect(":~") 360 linkCheck = main.ONOSbench.handle.before 361 if "links=16," in linkCheck: 362 break 363 time.sleep(1) 364 if i == 9: 365 main.log.info("Links Failed to reconnect, next iteration of data invalid." + linkCheck) 366 367 if run < warmUp: 368 main.log.info("Warm up run " + str(run+1) + " completed") 369 370 if debug: main.log.info(myResult) 371 latTemp = [] 372 nodeTemp = [] 373 for i in myResult: 374 latTemp.append(i[0]) 375 nodeTemp.append(i[1]) 376 377 mode = {} 378 for i in nodeTemp: 379 if i in mode: 380 mode[i] += 1 381 else: 382 mode[i] = 1 383 384 for i in mode.keys(): 385 if mode[i] == max(mode.values()): 386 nodeMode = i 387 388 average = numpy.average(latTemp) 389 stdDev = numpy.std(latTemp) 390 391 average = numpy.multiply(average, 1000) 392 stdDev = numpy.multiply(stdDev, 1000) 393 394 main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents)) 395 main.log.report("Latency average:................" + str(average)) 396 main.log.report("Latency standard deviation:....." + str(stdDev)) 397 main.log.report("Mode of last node to respond:..." + str(nodeMode)) 398 main.log.report("________________________________________________________") 399 400 resultsDB = open("/tmp/IntentRerouteLatDB", "a") 401 resultsDB.write("'" + commit + "',") 402 resultsDB.write(str(clusterCount) + ",") 403 resultsDB.write(str(intents) + ",") 404 resultsDB.write(str(average) + ",") 405 resultsDB.write(str(stdDev) + "\n") 406 resultsDB.close() 407 408 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
409