blob: 9127690d7160c355cf26480be203e6d64415eb36 [file] [log] [blame]
# ScaleOutTemplate
#
# CASE1 starts number of nodes specified in param file
#
# cameron@onlab.us
import sys
import os.path
class IntentRerouteLat:
def __init__( self ):
self.default = ''
def CASE1( self, main ):
import time
global init
try:
if type(init) is not bool:
init = False
except NameError:
init = False
#Load values from params file
checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
gitPull = main.params[ 'GIT' ][ 'autopull' ]
cellName = main.params[ 'ENV' ][ 'cellName' ]
Apps = main.params[ 'ENV' ][ 'cellApps' ]
BENCHUser = main.params[ 'BENCH' ][ 'user' ]
maxNodes = int(main.params[ 'availableNodes' ])
skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
cellName = main.params[ 'ENV' ][ 'cellName' ]
# -- INIT SECTION, ONLY RUNS ONCE -- #
if init == False:
init = True
global clusterCount #number of nodes running
global ONOSIp #list of ONOS IP addresses
global scale
global commit
clusterCount = 0
ONOSIp = [ 0 ]
scale = (main.params[ 'SCALE' ]).split(",")
clusterCount = int(scale[0])
#Populate ONOSIp with ips from params
ONOSIp = [0]
ONOSIp.extend(main.ONOSbench.getOnosIps())
print("-----------------" + str(ONOSIp))
#mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
if skipMvn != "yes":
mvnResult = main.ONOSbench.cleanInstall()
#git
main.step( "Git checkout and pull " + checkoutBranch )
if gitPull == 'on':
checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
pullResult = main.ONOSbench.gitPull()
else:
checkoutResult = main.TRUE
pullResult = main.TRUE
main.log.info( "Skipped git checkout and pull" )
commit = main.ONOSbench.getVersion()
commit = (commit.split(" "))[1]
resultsDB = open("IntentRerouteLatDB", "w+")
resultsDB.close()
# -- END OF INIT SECTION --#
clusterCount = int(scale[0])
scale.remove(scale[0])
MN1Ip = ONOSIp[len(ONOSIp)-1]
BENCHIp = ONOSIp[len(ONOSIp)-2]
#kill off all onos processes
main.log.step("Safety check, killing all ONOS processes")
main.log.step("before initiating enviornment setup")
for node in range(1, maxNodes + 1):
main.ONOSbench.onosDie(ONOSIp[node])
#Uninstall everywhere
main.log.step( "Cleaning Enviornment..." )
for i in range(1, maxNodes + 1):
main.log.info(" Uninstalling ONOS " + str(i) )
main.ONOSbench.onosUninstall( ONOSIp[i] )
#construct the cell file
main.log.info("Creating cell file")
cellIp = []
for node in range (1, clusterCount + 1):
cellIp.append(ONOSIp[node])
print "Cell ip" + str(cellIp)
print cellName
print MN1Ip
print Apps
main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), *cellIp)
main.step( "Set Cell" )
main.ONOSbench.setCell(cellName)
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
main.step( "verify cells" )
verifyCellResult = main.ONOSbench.verifyCell()
main.log.report( "Initializing " + str( clusterCount ) + " node cluster." )
for node in range(1, clusterCount + 1):
main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
main.ONOSbench.onosInstall( ONOSIp[node])
for node in range(1, clusterCount + 1):
for i in range( 2 ):
isup = main.ONOSbench.isup( ONOSIp[node] )
if isup:
main.log.info("ONOS " + str(node) + " is up\n")
break
if not isup:
main.log.report( "ONOS " + str(node) + " didn't start!" )
main.log.info("Startup sequence complete")
deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
print("Device mastership list: " + str(deviceMastership))
main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount 8 " """)
main.ONOSbench.handle.expect(":~")
print repr(main.ONOSbench.handle.before)
time.sleep(3)
main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape reroute" """)
main.ONOSbench.handle.expect(":~")
print repr(main.ONOSbench.handle.before)
time.sleep(3)
time.sleep(10)
once = True
for attempt in range(0,10):
main.ONOSbench.handle.sendline("onos $OC1 summary")
main.ONOSbench.handle.expect(":~")
x = main.ONOSbench.handle.before
main.log.info("Initial setup. ONOS summary response: \n" + x + "\n")
if "devices=8" in x and "links=16," in x:
break
else:
if once == False:
main.log.error("Null provider start failed, retrying..")
once = False
main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled false" """)
main.ONOSbench.handle.expect(":~")
time.sleep(3)
main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled true" """)
main.ONOSbench.handle.expect(":~")
time.sleep(8)
index = 1
for node in deviceMastership:
for attempt in range(0,10):
cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)] + """ master" """)
main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
main.ONOSbench.handle.sendline(cmd)
main.ONOSbench.handle.expect(":~")
time.sleep(4)
cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index))
main.log.info(cmd)
main.ONOSbench.handle.sendline(cmd)
main.ONOSbench.handle.expect(":~")
check = main.ONOSbench.handle.before
main.log.info("CHECK:\n" + check)
if ("master=" + ONOSIp[int(node)]) in check:
break
index += 1
main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
def CASE2( self, main ):
import time
import numpy
import datetime
#from scipy import stats
ts = time.time()
date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d')
sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
intentsList = (main.params[ 'TEST' ][ 'intents' ]).split(",")
debug = main.params[ 'TEST' ][ 'debug' ]
for i in range(0,len(intentsList)):
intentsList[i] = int(intentsList[i])
timestampMetrics = []
if main.params['METRICS']['Submitted'] == "1":
timestampMetrics.append("Submitted")
if main.params['METRICS']['Installed'] == "1":
timestampMetrics.append("Installed")
if main.params['METRICS']['Failed'] == "1":
timestampMetrics.append("Failed")
if main.params['METRICS']['Withdraw'] == "1":
timestampMetrics.append("Withdraw")
if main.params['METRICS']['Withdrawn'] == "1":
timestampMetrics.append("Withdrawn")
if debug: main.log.info(timestampMetrics)
if debug == "True":
debug = True
else:
debug = False
ingress = "null:0000000000000001"
egress = "null:0000000000000007"
for intents in intentsList:
main.log.report("Intent Batch size: " + str(intents) + "\n ")
myResult = [["latency", "lastNode"] for x in range(sampleSize)]
for run in range(0, (warmUp + sampleSize)):
if run > warmUp:
main.log.info("Starting test iteration " + str(run-warmUp))
cmd = """onos $OC1 "push-test-intents -i """
cmd += ingress + "/0 "
cmd += egress + "/0 "
cmd += str(intents) +""" 1" """
if debug: main.log.info(cmd)
withdrawCmd = cmd.replace("intents -i", "intents -w ")
#push-test-intents
main.ONOSbench.handle.sendline(cmd)
main.ONOSbench.handle.expect(":~")
myRawResult = main.ONOSbench.handle.before
for i in range(0, 40):
main.ONOSbench.handle.sendline("onos $OC1 summary")
main.ONOSbench.handle.expect(":~")
linkCheck = main.ONOSbench.handle.before
if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
break
if i == 39:
main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
#collect timestamp from link cut
cmd = """onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 down" """
if debug: main.log.info("COMMAND: " + str(cmd))
main.ONOSbench.handle.sendline(cmd)
main.ONOSbench.handle.expect(":~")
cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
for i in range(0,10):
main.ONOSbench.handle.sendline(cmd)
time.sleep(2)
main.ONOSbench.handle.expect(":~")
raw = main.ONOSbench.handle.before
#if "NullLinkProvider" in raw and "links=14" in raw:
if "links=14" in raw:
break
if i >= 9:
main.log.error("Expected output not being recieved... continuing")
main.log.info(raw)
break
time.sleep(2)
temp = raw.splitlines()
for line in temp:
if str(date) in line:
temp = line
break
cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1]
if debug: main.log.info("Cut timestamp: " + cutTimestamp)
#validate link count and flow count
for i in range(0, 40):
main.ONOSbench.handle.sendline("onos $OC1 summary")
main.ONOSbench.handle.expect(":~")
linkCheck = main.ONOSbench.handle.before
#if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
break
if i == 39:
main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
time.sleep(5) #trying to avoid negative values
#intents events metrics installed timestamp
IEMtimestamps = [0]*(clusterCount + 1)
installedTemp = [0]*(clusterCount + 1)
for node in range(1, clusterCount +1):
cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
raw = ""
while "epoch)" not in raw:
main.ONOSbench.handle.sendline(cmd)
main.ONOSbench.handle.expect(":~")
raw = main.ONOSbench.handle.before
print(raw)
intentsTimestamps = {}
rawTimestamps = raw.splitlines()
for line in rawTimestamps:
if "Timestamp" in line and "grep" not in line:
metricKey = (line.split(" "))[1]
metricTimestamp = (line.split(" ")[len(line.split(" ")) -1]).replace("epoch)=","")
metricTimestamp = float(metricTimestamp)
metricTimestamp = numpy.divide(metricTimestamp, 1000)
if debug: main.log.info(repr(metricTimestamp))
intentsTimestamps[metricKey] = metricTimestamp
if metricKey == "Installed":
installedTemp[node] = metricTimestamp
main.log.info("Node: " + str(node) + " Timestamps: " + str(intentsTimestamps))
IEMtimestamps[node] = intentsTimestamps
myMax = max(installedTemp)
indexOfMax = installedTemp.index(myMax)
#number crunch
for metric in timestampMetrics: #this is where we sould add support for computing other timestamp metrics
if metric == "Installed":
if run >= warmUp:
main.log.report("link cut timestamp: " + cutTimestamp)
#readableInstalledTimestamp = str(intentsTimestamps["Installed"])
readableInstalledTimestamp = str(myMax)
#main.log.report("Intent Installed timestamp: " + str(intentsTimestamps["Installed"]))
main.log.report("Intent Installed timestamp: " + str(myMax))
cutEpoch = time.mktime(time.strptime(cutTimestamp, "%Y-%m-%d %H:%M:%S,%f"))
if debug: main.log.info("cutEpoch=" + str(cutEpoch))
#rerouteLatency = float(intentsTimestamps["Installed"] - cutEpoch)
rerouteLatency = float(myMax - cutEpoch)
rerouteLatency = numpy.divide(rerouteLatency, 1000)
main.log.report("Reroute latency:" + str(rerouteLatency) + " (seconds)\n ")
myResult[run-warmUp][0] = rerouteLatency
myResult[run-warmUp][1] = indexOfMax
if debug: main.log.info("Latency: " + str(myResult[run-warmUp][0]))
if debug: main.log.info("last node: " + str(myResult[run-warmUp][1]))
cmd = """ onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 up" """
if debug: main.log.info(cmd)
main.ONOSbench.handle.sendline(cmd)
main.ONOSbench.handle.expect(":~")
#wait for intent withdraw
main.ONOSbench.handle.sendline(withdrawCmd)
main.log.info(withdrawCmd)
main.ONOSbench.handle.expect(":~")
if debug: main.log.info(main.ONOSbench.handle.before)
main.ONOSbench.handle.sendline("onos $OC1 intents|grep WITHDRAWN|wc -l")
main.ONOSbench.handle.expect(":~")
intentWithdrawCheck = main.ONOSbench.handle.before
if (str(intents)) in intentWithdrawCheck:
main.log.info("intents withdrawn")
if debug: main.log.info(intentWithdrawCheck)
# wait for links to be reestablished
for i in range(0, 10):
main.ONOSbench.handle.sendline("onos $OC1 summary")
main.ONOSbench.handle.expect(":~")
linkCheck = main.ONOSbench.handle.before
if "links=16," in linkCheck:
break
time.sleep(1)
if i == 9:
main.log.info("Links Failed to reconnect, next iteration of data invalid." + linkCheck)
if run < warmUp:
main.log.info("Warm up run " + str(run+1) + " completed")
if debug: main.log.info(myResult)
latTemp = []
nodeTemp = []
for i in myResult:
latTemp.append(i[0])
nodeTemp.append(i[1])
mode = {}
for i in nodeTemp:
if i in mode:
mode[i] += 1
else:
mode[i] = 1
for i in mode.keys():
if mode[i] == max(mode.values()):
nodeMode = i
average = numpy.average(latTemp)
stdDev = numpy.std(latTemp)
average = numpy.multiply(average, 1000)
stdDev = numpy.multiply(stdDev, 1000)
main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents))
main.log.report("Latency average:................" + str(average))
main.log.report("Latency standard deviation:....." + str(stdDev))
main.log.report("Mode of last node to respond:..." + str(nodeMode))
main.log.report("________________________________________________________")
resultsDB = open("IntentRerouteLatDB", "a")
resultsDB.write("'" + commit + "',")
resultsDB.write(str(clusterCount) + ",")
resultsDB.write(str(intents) + ",")
resultsDB.write(str(average) + ",")
resultsDB.write(str(stdDev) + "\n")
resultsDB.close()
main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])