Merge "Refactor IntentRerouteLatency test - Refactor IntentRerouteLatency and IntentRerouteLatWithFOBJ test - Using driver function to handle ONOS - Improve algorithm, fix a calculation bug"
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 3e94166..a3e2b6a 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -849,7 +849,7 @@
main.cleanup()
main.exit()
- def links( self, jsonFormat=True ):
+ def links( self, jsonFormat=True, timeout=30 ):
"""
Lists all core links
Optional argument:
@@ -859,7 +859,7 @@
cmdStr = "links"
if jsonFormat:
cmdStr += " -j"
- handle = self.sendline( cmdStr )
+ handle = self.sendline( cmdStr, timeout=timeout )
assert "Command not found:" not in handle, handle
return handle
except AssertionError:
@@ -1872,7 +1872,7 @@
main.cleanup()
main.exit()
- def removeAllIntents( self, purge=False, sync=False, app='org.onosproject.cli' ):
+ def removeAllIntents( self, purge=False, sync=False, app='org.onosproject.cli', timeout=30 ):
"""
Description:
Remove all the intents
@@ -1891,7 +1891,7 @@
cmdStr += " -s"
cmdStr += " " + app
- handle = self.sendline( cmdStr )
+ handle = self.sendline( cmdStr, timeout=timeout )
assert "Command not found:" not in handle, handle
if re.search( "Error", handle ):
main.log.error( "Error in removing intent" )
@@ -4386,7 +4386,7 @@
main.cleanup()
main.exit()
- def summary( self, jsonFormat=True ):
+ def summary( self, jsonFormat=True, timeout=30 ):
"""
Description: Execute summary command in onos
Returns: json object ( summary -j ), returns main.FALSE if there is
@@ -4397,7 +4397,7 @@
cmdStr = "summary"
if jsonFormat:
cmdStr += " -j"
- handle = self.sendline( cmdStr )
+ handle = self.sendline( cmdStr, timeout=timeout )
assert handle is not None, "Error in sendline"
assert "Command not found:" not in handle, handle
assert "Error:" not in handle, handle
@@ -4758,7 +4758,7 @@
main.cleanup()
main.exit()
- def link( self, begin, end, state ):
+ def link( self, begin, end, state, timeout=30, showResponse=True ):
'''
Description:
Bring link down or up in the null-provider.
@@ -4771,7 +4771,7 @@
'''
try:
cmd = "null-link null:{} null:{} {}".format( begin, end, state )
- response = self.sendline( cmd, showResponse=True )
+ response = self.sendline( cmd, showResponse=showResponse, timeout=timeout )
assert response is not None, "Error in sendline"
assert "Command not found:" not in response, response
if "Error" in response or "Failure" in response:
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 0b5906f..dde65d3 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -1664,6 +1664,8 @@
"stop/waiting",
"Not Running ...",
pexpect.TIMEOUT ], timeout=120 )
+ self.handle.sendline( "" )
+ self.handle.expect( "\$" )
if i == 0 or i == 1:
main.log.info( "ONOS is running" )
diff --git a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params
index 3799982..9dfa7a4 100644
--- a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params
+++ b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params
@@ -1,6 +1,6 @@
<PARAMS>
- <testcases>1,2,1,2,1,2,1,2</testcases>
+ <testcases>0,1,2,1,2,1,2,1,2</testcases>
<SCALE>1,3,5,7</SCALE>
<max>7</max>
@@ -14,30 +14,33 @@
<skipCleanInstall>yes</skipCleanInstall>
<warmUp>5</warmUp>
<sampleSize>20</sampleSize>
- <wait></wait>
<intents>1,100,1000</intents> #list format, will be split on ','
- <debug>True</debug>
-
- <s1>1,1,1,1,1,1,1,1</s1>
- <s3>2,2,1,1,3,3,3,1</s3>
- <s5>2,2,1,1,3,4,5,3</s5>
- <s7>2,3,1,1,5,6,7,4</s7>
-
+ <ingress>null:0000000000000001/0</ingress>
+ <egress>null:0000000000000007/0</egress>
+ <debug>False</debug>
</TEST>
-
- <METRICS>
- <Submitted>0</Submitted>
- <Installed>1</Installed>
- <Failed>0</Failed>
- <Withdraw>0</Withdraw>
- <Withdrawn>0</Withdrawn>
- </METRICS>
+ <DATABASE>
+ <file>/tmp/IntentRerouteLatDB</file>
+ </DATABASE>
<GIT>
- <autopull>off</autopull>
- <checkout>master</checkout>
+ <gitPull>False</gitPull>
+ <gitBranch>master</gitBranch>
</GIT>
+ <ATTEMPTS>
+ <verify>3</verify>
+ </ATTEMPTS>
+
+ <SLEEP>
+ <startup>10</startup>
+ <install>10</install>
+ <verify>3</verify>
+ <reroute>3</reroute>
+ # timeout for pexpect
+ <timeout>300</timeout>
+ </SLEEP>
+
<CTRL>
<USER>sdn</USER>
diff --git a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.py b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.py
index 0c4d950..182f4b6 100644
--- a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.py
+++ b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.py
@@ -1,419 +1,366 @@
-# ScaleOutTemplate
-#
-# CASE1 starts number of nodes specified in param file
-#
-# cameron@onlab.us
-
-import sys
-import os.path
-
+# SCPFintentRerouteLat
+"""
+SCPFintentRerouteLat
+ - Test Intent Reroute Latency
+ - Test Algorithm:
+ 1. Start Null Provider reroute Topology
+ 2. Using Push-test-intents to push batch size intents from switch 1 to switch 7
+ 3. Cut the link between switch 3 and switch 4 (the path will reroute to switch 8)
+ 4. Get the topology time stamp
+ 5. Get Intent reroute(Installed) time stamp from each nodes
+ 6. Use the latest intent time stamp subtract topology time stamp
+ - This test will run 5 warm up by default, warm up iteration can be setup in Param file
+ - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file
+ - The unit of the latency result is milliseconds
+"""
class SCPFintentRerouteLat:
-
- def __init__( self ):
+ def __init__(self):
self.default = ''
- def CASE1( self, main ):
+ def CASE0( self, main ):
+ '''
+ - GIT
+ - BUILDING ONOS
+ Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
+ This step is usually skipped. Because in a Jenkins driven automated
+ test env. We want Jenkins jobs to pull&build for flexibility to handle
+ different versions of ONOS.
+ - Construct tests variables
+ '''
+ gitPull = main.params['GIT']['gitPull']
+ gitBranch = main.params['GIT']['gitBranch']
+ main.case("Pull onos branch and build onos on Teststation.")
+
+ if gitPull == 'True':
+ main.step("Git Checkout ONOS branch: " + gitBranch)
+ stepResult = main.ONOSbench.gitCheckout(branch=gitBranch)
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully checkout onos branch.",
+ onfail="Failed to checkout onos branch. Exiting test...")
+ if not stepResult: main.exit()
+
+ main.step("Git Pull on ONOS branch:" + gitBranch)
+ stepResult = main.ONOSbench.gitPull()
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully pull onos. ",
+ onfail="Failed to pull onos. Exiting test ...")
+ if not stepResult: main.exit()
+
+ main.step("Building ONOS branch: " + gitBranch)
+ stepResult = main.ONOSbench.cleanInstall(skipTest=True)
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully build onos.",
+ onfail="Failed to build onos. Exiting test...")
+ if not stepResult: main.exit()
+
+ else:
+ main.log.warn("Skipped pulling onos and Skipped building ONOS")
+
+ main.apps = main.params['ENV']['cellApps']
+ main.BENCHUser = main.params['BENCH']['user']
+ main.BENCHIp = main.params['BENCH']['ip1']
+ main.MN1Ip = main.params['MN']['ip1']
+ main.maxNodes = int(main.params['max'])
+ main.skipMvn = main.params['TEST']['skipCleanInstall']
+ main.cellName = main.params['ENV']['cellName']
+ main.scale = (main.params['SCALE']).split(",")
+ main.dbFileName = main.params['DATABASE']['file']
+ main.timeout = int(main.params['SLEEP']['timeout'])
+ main.startUpSleep = int(main.params['SLEEP']['startup'])
+ main.installSleep = int(main.params['SLEEP']['install'])
+ main.verifySleep = int(main.params['SLEEP']['verify'])
+ main.verifyAttempts = int(main.params['ATTEMPTS']['verify'])
+ main.sampleSize = int(main.params['TEST']['sampleSize'])
+ main.warmUp = int(main.params['TEST']['warmUp'])
+ main.intentsList = (main.params['TEST']['intents']).split(",")
+ main.ingress = main.params['TEST']['ingress']
+ main.egress = main.params['TEST']['egress']
+ main.debug = main.params['TEST']['debug']
+ for i in range(0, len(main.intentsList)):
+ main.intentsList[i] = int(main.intentsList[i])
+ # Create DataBase file
+ main.log.info("Create Database file " + main.dbFileName)
+ resultsDB = open(main.dbFileName, "w+")
+ resultsDB.close()
+
+ def CASE1( self, main ):
+ '''
+ clean up test environment and set up
+ '''
import time
- global init
- try:
- if type(init) is not bool:
- init = False
- except NameError:
- init = False
+ main.log.info("Get ONOS cluster IP")
+ print(main.scale)
+ main.numCtrls = int(main.scale[0])
+ main.ONOSip = []
+ main.maxNumBatch = 0
+ main.AllONOSip = main.ONOSbench.getOnosIps()
+ for i in range(main.numCtrls):
+ main.ONOSip.append(main.AllONOSip[i])
+ main.log.info(main.ONOSip)
+ main.CLIs = []
+ main.log.info("Creating list of ONOS cli handles")
+ for i in range(main.numCtrls):
+ main.CLIs.append(getattr(main, 'ONOS%scli' % (i + 1)))
- #Load values from params file
- checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
- gitPull = main.params[ 'GIT' ][ 'autopull' ]
- cellName = main.params[ 'ENV' ][ 'cellName' ]
- Apps = main.params[ 'ENV' ][ 'cellApps' ]
- BENCHUser = main.params[ 'BENCH' ][ 'user' ]
- BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
- MN1Ip = main.params[ 'MN' ][ 'ip1' ]
- main.maxNodes = int(main.params[ 'max' ])
- skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
- cellName = main.params[ 'ENV' ][ 'cellName' ]
+ if not main.CLIs:
+ main.log.error("Failed to create the list of ONOS cli handles")
+ main.cleanup()
+ main.exit()
- # -- INIT SECTION, ONLY RUNS ONCE -- #
- if init == False:
- init = True
- global clusterCount #number of nodes running
- global ONOSIp #list of ONOS IP addresses
- global scale
- global commit
+ main.commit = main.ONOSbench.getVersion(report=True)
+ main.commit = main.commit.split(" ")[1]
+ main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls)
+ main.log.info("Safety check, killing all ONOS processes" +
+ " before initiating environment setup")
- clusterCount = 0
- ONOSIp = [ 0 ]
- scale = (main.params[ 'SCALE' ]).split(",")
- clusterCount = int(scale[0])
+ for i in range(main.numCtrls):
+ main.ONOSbench.onosDie(main.ONOSip[i])
- #Populate ONOSIp with ips from params
- ONOSIp = [0]
- ONOSIp.extend(main.ONOSbench.getOnosIps())
+ main.log.info("NODE COUNT = %s" % main.numCtrls)
- print("-----------------" + str(ONOSIp))
- #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
- if skipMvn != "yes":
- mvnResult = main.ONOSbench.cleanInstall()
+ # tempOnosIp = []
+ # for i in range(main.numCtrls):
+ # tempOnosIp.append(main.AllONOSip[i])
+ # print(tempOnosIp)
+ main.ONOSbench.createCellFile(main.ONOSbench.ip_address,
+ main.cellName,
+ main.MN1Ip,
+ main.apps,
+ main.ONOSip)
+ main.step("Apply cell to environment")
+ cellResult = main.ONOSbench.setCell(main.cellName)
+ verifyResult = main.ONOSbench.verifyCell()
+ stepResult = cellResult and verifyResult
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully applied cell to " + \
+ "environment",
+ onfail="Failed to apply cell to environment ")
- #git
- main.step( "Git checkout and pull " + checkoutBranch )
- if gitPull == 'on':
- checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
- pullResult = main.ONOSbench.gitPull()
-
- else:
- checkoutResult = main.TRUE
- pullResult = main.TRUE
- main.log.info( "Skipped git checkout and pull" )
-
- commit = main.ONOSbench.getVersion()
- commit = (commit.split(" "))[1]
-
- resultsDB = open("/tmp/IntentRerouteLatDB", "w+")
- resultsDB.close()
-
- # -- END OF INIT SECTION --#
-
- clusterCount = int(scale[0])
- scale.remove(scale[0])
-
- #kill off all onos processes
- main.log.step("Safety check, killing all ONOS processes")
- main.log.step("before initiating environment setup")
- for node in range(1, main.maxNodes + 1):
- main.ONOSbench.onosDie(ONOSIp[node])
-
- #Uninstall everywhere
- main.log.step( "Cleaning Enviornment..." )
- for i in range(1, main.maxNodes + 1):
- main.log.info(" Uninstalling ONOS " + str(i) )
- main.ONOSbench.onosUninstall( ONOSIp[i] )
-
- #construct the cell file
- main.log.info("Creating cell file")
- cellIp = []
- for node in range (1, clusterCount + 1):
- cellIp.append(ONOSIp[node])
-
- main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), cellIp)
-
- main.step( "Set Cell" )
- main.ONOSbench.setCell(cellName)
-
- main.step( "Creating ONOS package" )
+ main.step("Creating ONOS package")
packageResult = main.ONOSbench.onosPackage()
+ stepResult = packageResult
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully created ONOS package",
+ onfail="Failed to create ONOS package")
- main.step( "verify cells" )
- verifyCellResult = main.ONOSbench.verifyCell()
+ main.step("Uninstall ONOS package on all Nodes")
+ uninstallResult = main.TRUE
+ for i in range(int(main.numCtrls)):
+ main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i])
+ u_result = main.ONOSbench.onosUninstall(main.ONOSip[i])
+ utilities.assert_equals(expect=main.TRUE, actual=u_result,
+ onpass="Test step PASS",
+ onfail="Test step FAIL")
+ uninstallResult = (uninstallResult and u_result)
- main.log.report( "Initializing " + str( clusterCount ) + " node cluster." )
- for node in range(1, clusterCount + 1):
- main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
- main.ONOSbench.onosInstall( ONOSIp[node])
+ main.step("Install ONOS package on all Nodes")
+ installResult = main.TRUE
+ for i in range(int(main.numCtrls)):
+ main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i])
+ i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i])
+ utilities.assert_equals(expect=main.TRUE, actual=i_result,
+ onpass="Test step PASS",
+ onfail="Test step FAIL")
+ installResult = installResult and i_result
- for node in range(1, clusterCount + 1):
- for i in range( 2 ):
- isup = main.ONOSbench.isup( ONOSIp[node] )
- if isup:
- main.log.info("ONOS " + str(node) + " is up\n")
- break
- if not isup:
- main.log.report( "ONOS " + str(node) + " didn't start!" )
- main.log.info("Startup sequence complete")
+ main.step("Verify ONOS nodes UP status")
+ statusResult = main.TRUE
+ for i in range(int(main.numCtrls)):
+ main.log.info("ONOS Node " + main.ONOSip[i] + " status:")
+ onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
+ utilities.assert_equals(expect=main.TRUE, actual=onos_status,
+ onpass="Test step PASS",
+ onfail="Test step FAIL")
+ statusResult = (statusResult and onos_status)
+ time.sleep(2)
+ main.step("Start ONOS CLI on all nodes")
+ cliResult = main.TRUE
+ main.log.step(" Start ONOS cli using thread ")
+ startCliResult = main.TRUE
+ pool = []
+ main.threadID = 0
+ for i in range(int(main.numCtrls)):
+ t = main.Thread(target=main.CLIs[i].startOnosCli,
+ threadID=main.threadID,
+ name="startOnosCli",
+ args=[main.ONOSip[i]],
+ kwargs={"onosStartTimeout": main.timeout})
+ pool.append(t)
+ t.start()
+ main.threadID = main.threadID + 1
+ for t in pool:
+ t.join()
+ startCliResult = startCliResult and t.result
+ time.sleep(main.startUpSleep)
- deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
- print("Device mastership list: " + str(deviceMastership))
+ # configure apps
+ main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=8)
+ main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="reroute")
+ main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
+ main.CLIs[0].setCfg("org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled", value="false")
- main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled false")
+ time.sleep(main.startUpSleep)
- main.log.step("Setting up null provider")
- for i in range(3):
- main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "deviceCount 8")
- main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "topoShape reroute")
- main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "enabled true")
- time.sleep(5)
- main.ONOSbench.handle.sendline("onos $OC1 summary")
- main.ONOSbench.handle.expect(":~")
- x = main.ONOSbench.handle.before
- if "devices=8" in x and "links=16," in x:
- break
-
- index = 1
- for node in deviceMastership:
- for attempt in range(0,10):
- cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)] + """ master" """)
- main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
- time.sleep(4)
-
- cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index))
- main.log.info(cmd)
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
- check = main.ONOSbench.handle.before
- main.log.info("CHECK:\n" + check)
- if ("master=" + ONOSIp[int(node)]) in check:
- break
- index += 1
-
- main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
+ # Balance Master
+ main.CLIs[0].balanceMasters()
+ if len(main.ONOSip) > 1:
+ main.CLIs[0].deviceRole("null:0000000000000003", main.ONOSip[0])
+ main.CLIs[0].deviceRole("null:0000000000000004", main.ONOSip[0])
def CASE2( self, main ):
-
import time
import numpy
import datetime
- #from scipy import stats
+ import json
+ # from scipy import stats
ts = time.time()
-
- sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
- warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
- intentsList = (main.params[ 'TEST' ][ 'intents' ]).split(",")
- debug = main.params[ 'TEST' ][ 'debug' ]
- for i in range(0,len(intentsList)):
- intentsList[i] = int(intentsList[i])
-
- timestampMetrics = []
- if main.params['METRICS']['Submitted'] == "1":
- timestampMetrics.append("Submitted")
- if main.params['METRICS']['Installed'] == "1":
- timestampMetrics.append("Installed")
- if main.params['METRICS']['Failed'] == "1":
- timestampMetrics.append("Failed")
- if main.params['METRICS']['Withdraw'] == "1":
- timestampMetrics.append("Withdraw")
- if main.params['METRICS']['Withdrawn'] == "1":
- timestampMetrics.append("Withdrawn")
- if debug: main.log.info(timestampMetrics)
-
- if debug == "True":
- debug = True
- else:
- debug = False
-
- ingress = "null:0000000000000001"
- egress = "null:0000000000000007"
-
- for intents in intentsList:
- main.log.report("Intent Batch size: " + str(intents) + "\n ")
- myResult = [["latency", "lastNode"] for x in range(sampleSize)]
-
- for run in range(0, (warmUp + sampleSize)):
- if run > warmUp:
- main.log.info("Starting test iteration " + str(run-warmUp))
-
- cmd = """onos $OC1 "push-test-intents -i """
- cmd += ingress + "/0 "
- cmd += egress + "/0 "
- cmd += str(intents) +""" 1" """
- if debug: main.log.info(cmd)
-
- withdrawCmd = cmd.replace("intents -i", "intents -w ")
-
- #push-test-intents
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
- myRawResult = main.ONOSbench.handle.before
-
- for i in range(0, 40):
- main.ONOSbench.handle.sendline("onos $OC1 summary")
- main.ONOSbench.handle.expect(":~")
- linkCheck = main.ONOSbench.handle.before
- if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
- break
- if i == 39:
- main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
- main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
- #main.ONOSbench.logReport(ONOSIp[(clusterCount-1)], ["ERROR", "WARNING", "EXCEPT"], "d")
- main.ONOSbench.sendline("onos $OC1 summary")
- main.ONOSbench.sendline("onos $OC1 devices")
- main.ONOSbench.sendline("onos $OC1 links")
- main.ONOSbench.expect(":~")
- main.log.info(main.ONOSbench.before)
-
- #collect timestamp from link cut
- cmd = """onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 down" """
- if debug: main.log.info("COMMAND: " + str(cmd))
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
-
- cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
- for i in range(0,10):
- main.ONOSbench.handle.sendline(cmd)
- time.sleep(2)
- main.ONOSbench.handle.expect(":~")
- raw = main.ONOSbench.handle.before
- #if "NullLinkProvider" in raw and "links=14" in raw:
- if "links=14" in raw:
- break
- if i >= 9:
- main.log.error("Expected output not being recieved... continuing")
- main.log.info(raw)
- break
- time.sleep(2)
-
- if debug: main.log.debug("raw: " + raw)
-
- temp = raw.splitlines()
-
- if debug: main.log.debug("temp (after splitlines): " + str(temp))
-
- # Since the string is deterministic the date is always the 3rd element.
- # However, if the data were grepping for in the onos log changes then this will
- # not work. This is why we print out the raw and temp string so we can visually
- # check if everything is in the correct order. temp should like this:
- # temp = ['/onos$ onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep Top ',
- # 'ologyManager| tail -1', '2015-10-15 12:03:33,736 ... ]
- temp = temp[2]
-
- if debug: main.log.debug("temp (checking for date): " + str(temp))
-
- cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1]
-
- if debug: main.log.info("Cut timestamp: " + cutTimestamp)
-
- #validate link count and flow count
- for i in range(0, 40):
- main.ONOSbench.handle.sendline("onos $OC1 summary")
- main.ONOSbench.handle.expect(":~")
- linkCheck = main.ONOSbench.handle.before
- #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
- if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
- break
- if i == 39:
- main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
- main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
-
- time.sleep(5) #trying to avoid negative values
-
- #intents events metrics installed timestamp
- IEMtimestamps = [0]*(clusterCount + 1)
- installedTemp = [0]*(clusterCount + 1)
- for node in range(1, clusterCount +1):
- cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
- raw = ""
- while "epoch)" not in raw:
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
- raw = main.ONOSbench.handle.before
-
- print(raw)
-
- intentsTimestamps = {}
- rawTimestamps = raw.splitlines()
- for line in rawTimestamps:
- if "Timestamp" in line and "grep" not in line:
- metricKey = (line.split(" "))[1]
- metricTimestamp = (line.split(" ")[len(line.split(" ")) -1]).replace("epoch)=","")
- metricTimestamp = float(metricTimestamp)
- metricTimestamp = numpy.divide(metricTimestamp, 1000)
- if debug: main.log.info(repr(metricTimestamp))
- intentsTimestamps[metricKey] = metricTimestamp
- if metricKey == "Installed":
- installedTemp[node] = metricTimestamp
-
- main.log.info("Node: " + str(node) + " Timestamps: " + str(intentsTimestamps))
- IEMtimestamps[node] = intentsTimestamps
-
- myMax = max(installedTemp)
- indexOfMax = installedTemp.index(myMax)
-
- #number crunch
- for metric in timestampMetrics: #this is where we sould add support for computing other timestamp metrics
- if metric == "Installed":
- if run >= warmUp:
- main.log.report("link cut timestamp: " + cutTimestamp)
- #readableInstalledTimestamp = str(intentsTimestamps["Installed"])
- readableInstalledTimestamp = str(myMax)
-
- #main.log.report("Intent Installed timestamp: " + str(intentsTimestamps["Installed"]))
- main.log.report("Intent Installed timestamp: " + str(myMax))
-
- cutEpoch = time.mktime(time.strptime(cutTimestamp, "%Y-%m-%d %H:%M:%S,%f"))
- if debug: main.log.info("cutEpoch=" + str(cutEpoch))
- #rerouteLatency = float(intentsTimestamps["Installed"] - cutEpoch)
- rerouteLatency = float(myMax - cutEpoch)
-
- rerouteLatency = numpy.divide(rerouteLatency, 1000)
- main.log.report("Reroute latency:" + str(rerouteLatency) + " (seconds)\n ")
- myResult[run-warmUp][0] = rerouteLatency
- myResult[run-warmUp][1] = indexOfMax
- if debug: main.log.info("Latency: " + str(myResult[run-warmUp][0]))
- if debug: main.log.info("last node: " + str(myResult[run-warmUp][1]))
-
- cmd = """ onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 up" """
- if debug: main.log.info(cmd)
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
-
- #wait for intent withdraw
- main.ONOSbench.handle.sendline(withdrawCmd)
- main.log.info(withdrawCmd)
- main.ONOSbench.handle.expect(":~")
- if debug: main.log.info(main.ONOSbench.handle.before)
- main.ONOSbench.handle.sendline("onos $OC1 intents|grep WITHDRAWN|wc -l")
- main.ONOSbench.handle.expect(":~")
- intentWithdrawCheck = main.ONOSbench.handle.before
- if (str(intents)) in intentWithdrawCheck:
- main.log.info("intents withdrawn")
- if debug: main.log.info(intentWithdrawCheck)
-
- # wait for links to be reestablished
- for i in range(0, 10):
- main.ONOSbench.handle.sendline("onos $OC1 summary")
- main.ONOSbench.handle.expect(":~")
- linkCheck = main.ONOSbench.handle.before
- if "links=16," in linkCheck:
- break
- time.sleep(1)
- if i == 9:
- main.log.info("Links Failed to reconnect, next iteration of data invalid." + linkCheck)
-
- if run < warmUp:
- main.log.info("Warm up run " + str(run+1) + " completed")
-
- if debug: main.log.info(myResult)
- latTemp = []
- nodeTemp = []
- for i in myResult:
- latTemp.append(i[0])
- nodeTemp.append(i[1])
-
- mode = {}
- for i in nodeTemp:
- if i in mode:
- mode[i] += 1
+ print(main.intentsList)
+ for batchSize in main.intentsList:
+ main.log.report("Intent Batch size: " + str(batchSize) + "\n ")
+ main.LatencyList = []
+ for run in range(0, (main.warmUp + main.sampleSize)):
+ if run >= main.warmUp:
+ main.log.info("================================================")
+ main.log.info("Starting test iteration " + str(run - main.warmUp))
+ main.log.info("================================================")
else:
- mode[i] = 1
+ main.log.info("====================Warm Up=====================")
- for i in mode.keys():
- if mode[i] == max(mode.values()):
- nodeMode = i
+ # push intents
+ main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
+ offset=1, options="-i", timeout=main.timeout)
- average = numpy.average(latTemp)
- stdDev = numpy.std(latTemp)
+ # check links and flows
+ k = 0
+ verify = main.FALSE
+ linkCheck = 0
+ flowsCheck = 0
+ while k <= main.verifyAttempts:
+ time.sleep(main.verifySleep)
+ summary = json.loads(main.CLIs[0].summary(timeout=main.timeout))
+ linkCheck = summary.get("links")
+ flowsCheck = summary.get("flows")
+ if linkCheck == 16 and flowsCheck == batchSize * 7:
+ main.log.info("links: {}, flows: {} ".format(linkCheck, flowsCheck))
+ verify = main.TRUE
+ break
+ k += 1
+ if not verify:
+ main.log.warn("Links or flows number are not match!")
+ main.log.warn("links: {}, flows: {} ".format(linkCheck, flowsCheck))
+ continue
- average = numpy.multiply(average, 1000)
- stdDev = numpy.multiply(stdDev, 1000)
+ # Bring link down
+ main.CLIs[0].link("0000000000000004/1", "0000000000000003/2", "down",
+ timeout=main.timeout, showResponse=False)
+ verify = main.FALSE
+ k = 0
+ topoManagerLog = ""
+ while k <= main.verifyAttempts:
+ time.sleep(main.verifySleep)
+ summary = json.loads(main.CLIs[0].summary(timeout=main.timeout))
+ linkCheck = summary.get("links")
+ flowsCheck = summary.get("flows")
+ if linkCheck == 14:
+ main.log.info("links: {}, flows: {} ".format(linkCheck, flowsCheck))
+ verify = main.TRUE
+ break
+ k += 1
+ if not verify:
+ main.log.warn("Links number are not match in TopologyManager log!")
+ main.log.warn(topoManagerLog)
+ continue
- main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents))
- main.log.report("Latency average:................" + str(average))
- main.log.report("Latency standard deviation:....." + str(stdDev))
- main.log.report("Mode of last node to respond:..." + str(nodeMode))
+ try:
+ # expect twice to clean the pexpect buffer
+ main.ONOSbench.handle.sendline("")
+ main.ONOSbench.handle.expect("\$")
+ main.ONOSbench.handle.expect("\$")
+ # send line by using bench, can't use driver because pexpect buffer problem
+ cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
+ main.ONOSbench.handle.sendline(cmd)
+ time.sleep(1)
+ main.ONOSbench.handle.expect(":~")
+ topoManagerLog = main.ONOSbench.handle.before
+ topoManagerLogTemp = topoManagerLog.splitlines()
+ # To make sure we get correct topology log
+ for lines in topoManagerLogTemp:
+ if "creationTime" in lines:
+ topoManagerLog = lines
+ main.log.info("Topology Manager log:")
+ print(topoManagerLog)
+ cutTimestamp = float(topoManagerLog.split("creationTime=")[1].split(",")[0])
+ except:
+ main.log.error("Topology Log is not correct!")
+ print(topoManagerLog)
+ # If we got wrong Topology log, we should skip this iteration, and continue for next one
+ continue
+
+ installedTemp = []
+ time.sleep(1)
+ for cli in main.CLIs:
+ tempJson = json.loads(cli.intentsEventsMetrics())
+ Installedtime = tempJson.get('intentInstalledTimestamp').get('value')
+ installedTemp.append(float(Installedtime))
+ for i in range(0, len(installedTemp)):
+ main.log.info("ONOS Node {} Installed Time stemp: {}".format((i + 1), installedTemp[i]))
+ maxInstallTime = float(max(installedTemp))
+ if run >= main.warmUp:
+ main.log.info("Installed time stemp: {0:f}".format(maxInstallTime))
+ main.log.info("CutTimestamp: {0:f}".format(cutTimestamp))
+ # Both timeStemps are milliseconds
+ main.log.info("Latency: {0:f}".format(float(maxInstallTime - cutTimestamp)))
+ main.LatencyList.append(float(maxInstallTime - cutTimestamp))
+
+ # Verify Summary after we bring up link, and withdrawn intents
+ main.CLIs[0].link("0000000000000004/1", "0000000000000003/2", "up",
+ timeout=main.timeout)
+ k = 0
+ verify = main.FALSE
+ linkCheck = 0
+ flowsCheck = 0
+ while k <= main.verifyAttempts:
+ time.sleep(main.verifySleep)
+ main.CLIs[0].removeAllIntents(purge=True, sync=True, timeout=main.timeout)
+ time.sleep(1)
+ main.CLIs[0].purgeWithdrawnIntents()
+ summary = json.loads(main.CLIs[0].summary())
+ linkCheck = summary.get("links")
+ flowsCheck = summary.get("flows")
+ intentCheck = summary.get("intents")
+ if linkCheck == 16 and flowsCheck == 0 and intentCheck == 0:
+ main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
+ verify = main.TRUE
+ break
+ k += 1
+ if not verify:
+ main.log.error("links, flows, or intents are not correct!")
+ main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
+ continue
+
+ aveLatency = 0
+ stdLatency = 0
+ aveLatency = numpy.average(main.LatencyList)
+ stdLatency = numpy.std(main.LatencyList)
+ main.log.report("Scale: " + str(main.numCtrls) + " \tIntent batch: " + str(batchSize))
+ main.log.report("Latency average:................" + str(aveLatency))
+ main.log.report("Latency standard deviation:....." + str(stdLatency))
main.log.report("________________________________________________________")
- resultsDB = open("/tmp/IntentRerouteLatDB", "a")
- resultsDB.write("'" + commit + "',")
- resultsDB.write(str(clusterCount) + ",")
- resultsDB.write(str(intents) + ",")
- resultsDB.write(str(average) + ",")
- resultsDB.write(str(stdDev) + "\n")
+ resultsDB = open(main.dbFileName, "a")
+ resultsDB.write("'" + main.commit + "',")
+ resultsDB.write(str(main.numCtrls) + ",")
+ resultsDB.write(str(batchSize) + ",")
+ resultsDB.write(str(aveLatency) + ",")
+ resultsDB.write(str(stdLatency) + "\n")
resultsDB.close()
-
- main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
-
+ del main.scale[0]
diff --git a/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.params b/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.params
index 3799982..35fbefb 100644
--- a/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.params
+++ b/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.params
@@ -1,6 +1,6 @@
<PARAMS>
- <testcases>1,2,1,2,1,2,1,2</testcases>
+ <testcases>0,1,2,1,2,1,2,1,2</testcases>
<SCALE>1,3,5,7</SCALE>
<max>7</max>
@@ -14,30 +14,33 @@
<skipCleanInstall>yes</skipCleanInstall>
<warmUp>5</warmUp>
<sampleSize>20</sampleSize>
- <wait></wait>
<intents>1,100,1000</intents> #list format, will be split on ','
- <debug>True</debug>
-
- <s1>1,1,1,1,1,1,1,1</s1>
- <s3>2,2,1,1,3,3,3,1</s3>
- <s5>2,2,1,1,3,4,5,3</s5>
- <s7>2,3,1,1,5,6,7,4</s7>
-
+ <ingress>null:0000000000000001/0</ingress>
+ <egress>null:0000000000000007/0</egress>
+ <debug>False</debug>
</TEST>
-
- <METRICS>
- <Submitted>0</Submitted>
- <Installed>1</Installed>
- <Failed>0</Failed>
- <Withdraw>0</Withdraw>
- <Withdrawn>0</Withdrawn>
- </METRICS>
+ <DATABASE>
+ <file>/tmp/IntentRerouteLatDB</file>
+ </DATABASE>
<GIT>
- <autopull>off</autopull>
- <checkout>master</checkout>
+ <gitPull>False</gitPull>
+ <gitBranch>master</gitBranch>
</GIT>
+ <ATTEMPTS>
+ <verify>3</verify>
+ </ATTEMPTS>
+
+ <SLEEP>
+ <startup>10</startup>
+ <install>10</install>
+ <verify>5</verify>
+ <reroute>3</reroute>
+ # timeout for pexpect
+ <timeout>300</timeout>
+ </SLEEP>
+
<CTRL>
<USER>sdn</USER>
diff --git a/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.py b/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.py
index ff158b7..ce9155d 100644
--- a/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.py
+++ b/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.py
@@ -1,419 +1,369 @@
-# ScaleOutTemplate
-#
-# CASE1 starts number of nodes specified in param file
-#
-# cameron@onlab.us
-
-import sys
-import os.path
-
+# SCPFintentRerouteLatWithFlowObj
+"""
+SCPFintentRerouteLat
+ - Test Intent Reroute Latency
+ - Test Algorithm:
+ 1. Start Null Provider reroute Topology
+ 2. Using Push-test-intents to push batch size intents from switch 1 to switch 7
+ 3. Cut the link between switch 3 and switch 4 (the path will reroute to switch 8)
+ 4. Get the topology time stamp
+ 5. Get Intent reroute(Installed) time stamp from each nodes
+ 6. Use the latest intent time stamp subtract topology time stamp
+ - This test will run 5 warm up by default, warm up iteration can be setup in Param file
+ - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file
+ - The unit of the latency result is milliseconds
+ - Ues flowObject set to True
+"""
class SCPFintentRerouteLatWithFlowObj:
+ def __init__(self):
+ self.default=''
- def __init__( self ):
- self.default = ''
+ def CASE0( self, main ):
+ '''
+ - GIT
+ - BUILDING ONOS
+ Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
+ This step is usually skipped. Because in a Jenkins driven automated
+ test env. We want Jenkins jobs to pull&build for flexibility to handle
+ different versions of ONOS.
+ - Construct tests variables
+ '''
+ gitPull = main.params['GIT']['gitPull']
+ gitBranch = main.params['GIT']['gitBranch']
+
+ main.case("Pull onos branch and build onos on Teststation.")
+
+ if gitPull == 'True':
+ main.step("Git Checkout ONOS branch: " + gitBranch)
+ stepResult = main.ONOSbench.gitCheckout(branch=gitBranch)
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully checkout onos branch.",
+ onfail="Failed to checkout onos branch. Exiting test...")
+ if not stepResult: main.exit()
+
+ main.step("Git Pull on ONOS branch:" + gitBranch)
+ stepResult = main.ONOSbench.gitPull()
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully pull onos. ",
+ onfail="Failed to pull onos. Exiting test ...")
+ if not stepResult: main.exit()
+
+ main.step("Building ONOS branch: " + gitBranch)
+ stepResult = main.ONOSbench.cleanInstall(skipTest=True)
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully build onos.",
+ onfail="Failed to build onos. Exiting test...")
+ if not stepResult: main.exit()
+
+ else:
+ main.log.warn("Skipped pulling onos and Skipped building ONOS")
+
+ main.apps = main.params['ENV']['cellApps']
+ main.BENCHUser = main.params['BENCH']['user']
+ main.BENCHIp = main.params['BENCH']['ip1']
+ main.MN1Ip = main.params['MN']['ip1']
+ main.maxNodes = int(main.params['max'])
+ main.skipMvn = main.params['TEST']['skipCleanInstall']
+ main.cellName = main.params['ENV']['cellName']
+ main.scale = (main.params['SCALE']).split(",")
+ main.dbFileName = main.params['DATABASE']['file']
+ main.timeout = int(main.params['SLEEP']['timeout'])
+ main.startUpSleep = int(main.params['SLEEP']['startup'])
+ main.installSleep = int(main.params['SLEEP']['install'])
+ main.verifySleep = int(main.params['SLEEP']['verify'])
+ main.verifyAttempts = int(main.params['ATTEMPTS']['verify'])
+ main.sampleSize = int(main.params['TEST']['sampleSize'])
+ main.warmUp = int(main.params['TEST']['warmUp'])
+ main.intentsList = (main.params['TEST']['intents']).split(",")
+ main.ingress = main.params['TEST']['ingress']
+ main.egress = main.params['TEST']['egress']
+ main.debug = main.params['TEST']['debug']
+ for i in range ( 0, len(main.intentsList) ):
+ main.intentsList[i] = int(main.intentsList[i])
+ # Create DataBase file
+ main.log.info( "Create Database file " + main.dbFileName )
+ resultsDB = open( main.dbFileName, "w+")
+ resultsDB.close()
+
def CASE1( self, main ):
-
+ '''
+ clean up test environment and set up
+ '''
import time
- global init
- try:
- if type(init) is not bool:
- init = False
- except NameError:
- init = False
+ main.log.info( "Get ONOS cluster IP" )
+ print(main.scale)
+ main.numCtrls = int(main.scale[0])
+ main.ONOSip = []
+ main.maxNumBatch = 0
+ main.AllONOSip = main.ONOSbench.getOnosIps()
+ for i in range(main.numCtrls):
+ main.ONOSip.append(main.AllONOSip[i])
+ main.log.info(main.ONOSip)
+ main.CLIs=[]
+ main.log.info("Creating list of ONOS cli handles")
+ for i in range(main.numCtrls):
+ main.CLIs.append(getattr(main, 'ONOS%scli' % (i + 1)))
- #Load values from params file
- checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
- gitPull = main.params[ 'GIT' ][ 'autopull' ]
- cellName = main.params[ 'ENV' ][ 'cellName' ]
- Apps = main.params[ 'ENV' ][ 'cellApps' ]
- BENCHUser = main.params[ 'BENCH' ][ 'user' ]
- BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
- MN1Ip = main.params[ 'MN' ][ 'ip1' ]
- main.maxNodes = int(main.params[ 'max' ])
- skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
- cellName = main.params[ 'ENV' ][ 'cellName' ]
+ if not main.CLIs:
+ main.log.error("Failed to create the list of ONOS cli handles")
+ main.cleanup()
+ main.exit()
- # -- INIT SECTION, ONLY RUNS ONCE -- #
- if init == False:
- init = True
- global clusterCount #number of nodes running
- global ONOSIp #list of ONOS IP addresses
- global scale
- global commit
+ main.commit = main.ONOSbench.getVersion(report=True)
+ main.commit = main.commit.split(" ")[1]
+ main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls)
+ main.log.info("Safety check, killing all ONOS processes" +
+ " before initiating environment setup")
- clusterCount = 0
- ONOSIp = [ 0 ]
- scale = (main.params[ 'SCALE' ]).split(",")
- clusterCount = int(scale[0])
+ for i in range(main.numCtrls):
+ main.ONOSbench.onosDie(main.ONOSip[i])
- #Populate ONOSIp with ips from params
- ONOSIp = [0]
- ONOSIp.extend(main.ONOSbench.getOnosIps())
+ main.log.info("NODE COUNT = %s" % main.numCtrls)
- print("-----------------" + str(ONOSIp))
- #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
- if skipMvn != "yes":
- mvnResult = main.ONOSbench.cleanInstall()
+ #tempOnosIp = []
+ #for i in range(main.numCtrls):
+ # tempOnosIp.append(main.AllONOSip[i])
+ #print(tempOnosIp)
+ main.ONOSbench.createCellFile(main.ONOSbench.ip_address,
+ main.cellName,
+ main.MN1Ip,
+ main.apps,
+ main.ONOSip )
+ main.step("Apply cell to environment")
+ cellResult = main.ONOSbench.setCell(main.cellName)
+ verifyResult = main.ONOSbench.verifyCell()
+ stepResult = cellResult and verifyResult
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully applied cell to " + \
+ "environment",
+ onfail="Failed to apply cell to environment ")
- #git
- main.step( "Git checkout and pull " + checkoutBranch )
- if gitPull == 'on':
- checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
- pullResult = main.ONOSbench.gitPull()
-
- else:
- checkoutResult = main.TRUE
- pullResult = main.TRUE
- main.log.info( "Skipped git checkout and pull" )
-
- commit = main.ONOSbench.getVersion()
- commit = (commit.split(" "))[1]
-
- resultsDB = open("/tmp/IntentRerouteLatDBWithFlowObj", "w+")
- resultsDB.close()
-
- # -- END OF INIT SECTION --#
-
- clusterCount = int(scale[0])
- scale.remove(scale[0])
-
- #kill off all onos processes
- main.log.step("Safety check, killing all ONOS processes")
- main.log.step("before initiating environment setup")
- for node in range(1, main.maxNodes + 1):
- main.ONOSbench.onosDie(ONOSIp[node])
-
- #Uninstall everywhere
- main.log.step( "Cleaning Enviornment..." )
- for i in range(1, main.maxNodes + 1):
- main.log.info(" Uninstalling ONOS " + str(i) )
- main.ONOSbench.onosUninstall( ONOSIp[i] )
-
- #construct the cell file
- main.log.info("Creating cell file")
- cellIp = []
- for node in range (1, clusterCount + 1):
- cellIp.append(ONOSIp[node])
-
- main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), cellIp)
-
- main.step( "Set Cell" )
- main.ONOSbench.setCell(cellName)
-
- main.step( "Creating ONOS package" )
+ main.step("Creating ONOS package")
packageResult = main.ONOSbench.onosPackage()
+ stepResult = packageResult
+ utilities.assert_equals(expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully created ONOS package",
+ onfail="Failed to create ONOS package")
- main.step( "verify cells" )
- verifyCellResult = main.ONOSbench.verifyCell()
+ main.step("Uninstall ONOS package on all Nodes")
+ uninstallResult = main.TRUE
+ for i in range(int(main.numCtrls)):
+ main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i])
+ u_result = main.ONOSbench.onosUninstall(main.ONOSip[i])
+ utilities.assert_equals(expect=main.TRUE, actual=u_result,
+ onpass="Test step PASS",
+ onfail="Test step FAIL")
+ uninstallResult = (uninstallResult and u_result)
- main.log.report( "Initializing " + str( clusterCount ) + " node cluster." )
- for node in range(1, clusterCount + 1):
- main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
- main.ONOSbench.onosInstall( ONOSIp[node])
+ main.step("Install ONOS package on all Nodes")
+ installResult = main.TRUE
+ for i in range(int(main.numCtrls)):
+ main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i])
+ i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i])
+ utilities.assert_equals(expect=main.TRUE, actual=i_result,
+ onpass="Test step PASS",
+ onfail="Test step FAIL")
+ installResult = installResult and i_result
- for node in range(1, clusterCount + 1):
- for i in range( 2 ):
- isup = main.ONOSbench.isup( ONOSIp[node] )
- if isup:
- main.log.info("ONOS " + str(node) + " is up\n")
- break
- if not isup:
- main.log.report( "ONOS " + str(node) + " didn't start!" )
- main.log.info("Startup sequence complete")
+ main.step("Verify ONOS nodes UP status")
+ statusResult = main.TRUE
+ for i in range(int(main.numCtrls)):
+ main.log.info("ONOS Node " + main.ONOSip[i] + " status:")
+ onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
+ utilities.assert_equals(expect=main.TRUE, actual=onos_status,
+ onpass="Test step PASS",
+ onfail="Test step FAIL")
+ statusResult = (statusResult and onos_status)
+ time.sleep(2)
+ main.step("Start ONOS CLI on all nodes")
+ cliResult = main.TRUE
+ main.log.step(" Start ONOS cli using thread ")
+ startCliResult = main.TRUE
+ pool = []
+ main.threadID=0
+ for i in range(int(main.numCtrls)):
+ t = main.Thread(target=main.CLIs[i].startOnosCli,
+ threadID=main.threadID,
+ name="startOnosCli",
+ args=[main.ONOSip[i]],
+ kwargs={"onosStartTimeout": main.timeout})
+ pool.append(t)
+ t.start()
+ main.threadID = main.threadID + 1
+ for t in pool:
+ t.join()
+ startCliResult = startCliResult and t.result
+ time.sleep(main.startUpSleep)
- deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
- print("Device mastership list: " + str(deviceMastership))
+ # configure apps
+ main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=8)
+ main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="reroute")
+ main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
+ main.CLIs[0].setCfg("org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled", value="false")
+ main.CLIs[0].setCfg("org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator",
+ "useFlowObjectives", value="true")
+ time.sleep(main.startUpSleep)
- main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled false")
+ # Balance Master
+ main.CLIs[0].balanceMasters()
+ if len(main.ONOSip) > 1:
+ main.CLIs[0].deviceRole("null:0000000000000003", main.ONOSip[0])
+ main.CLIs[0].deviceRole("null:0000000000000004", main.ONOSip[0])
- main.log.step("Setting up null provider")
- for i in range(3):
- main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "deviceCount 8")
- main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "topoShape reroute")
- main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.provider.nil.NullProviders", "enabled true")
- main.ONOSbench.onosCfgSet( ONOSIp[1], "org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator", "useFlowObjectives true" )
- time.sleep(5)
- main.ONOSbench.handle.sendline("onos $OC1 summary")
- main.ONOSbench.handle.expect(":~")
- x = main.ONOSbench.handle.before
- if "devices=8" in x and "links=16," in x:
- break
-
- index = 1
- for node in deviceMastership:
- for attempt in range(0,10):
- cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)] + """ master" """)
- main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
- time.sleep(4)
-
- cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index))
- main.log.info(cmd)
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
- check = main.ONOSbench.handle.before
- main.log.info("CHECK:\n" + check)
- if ("master=" + ONOSIp[int(node)]) in check:
- break
- index += 1
-
- main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
-
- def CASE2( self, main ):
-
+ def CASE2( self, main):
import time
import numpy
import datetime
- #from scipy import stats
+ import json
+ # from scipy import stats
ts = time.time()
-
- sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
- warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
- intentsList = (main.params[ 'TEST' ][ 'intents' ]).split(",")
- debug = main.params[ 'TEST' ][ 'debug' ]
- for i in range(0,len(intentsList)):
- intentsList[i] = int(intentsList[i])
-
- timestampMetrics = []
- if main.params['METRICS']['Submitted'] == "1":
- timestampMetrics.append("Submitted")
- if main.params['METRICS']['Installed'] == "1":
- timestampMetrics.append("Installed")
- if main.params['METRICS']['Failed'] == "1":
- timestampMetrics.append("Failed")
- if main.params['METRICS']['Withdraw'] == "1":
- timestampMetrics.append("Withdraw")
- if main.params['METRICS']['Withdrawn'] == "1":
- timestampMetrics.append("Withdrawn")
- if debug: main.log.info(timestampMetrics)
-
- if debug == "True":
- debug = True
- else:
- debug = False
-
- ingress = "null:0000000000000001"
- egress = "null:0000000000000007"
-
- for intents in intentsList:
- main.log.report("Intent Batch size: " + str(intents) + "\n ")
- myResult = [["latency", "lastNode"] for x in range(sampleSize)]
-
- for run in range(0, (warmUp + sampleSize)):
- if run > warmUp:
- main.log.info("Starting test iteration " + str(run-warmUp))
-
- cmd = """onos $OC1 "push-test-intents -i """
- cmd += ingress + "/0 "
- cmd += egress + "/0 "
- cmd += str(intents) +""" 1" """
- if debug: main.log.info(cmd)
-
- withdrawCmd = cmd.replace("intents -i", "intents -w ")
-
- #push-test-intents
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
- myRawResult = main.ONOSbench.handle.before
-
- for i in range(0, 40):
- main.ONOSbench.handle.sendline("onos $OC1 summary")
- main.ONOSbench.handle.expect(":~")
- linkCheck = main.ONOSbench.handle.before
- if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
- break
- if i == 39:
- main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
- main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
- #main.ONOSbench.logReport(ONOSIp[(clusterCount-1)], ["ERROR", "WARNING", "EXCEPT"], "d")
- main.ONOSbench.sendline("onos $OC1 summary")
- main.ONOSbench.sendline("onos $OC1 devices")
- main.ONOSbench.sendline("onos $OC1 links")
- main.ONOSbench.expect(":~")
- main.log.info(main.ONOSbench.before)
-
- #collect timestamp from link cut
- cmd = """onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 down" """
- if debug: main.log.info("COMMAND: " + str(cmd))
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
-
- cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
- for i in range(0,10):
- main.ONOSbench.handle.sendline(cmd)
- time.sleep(2)
- main.ONOSbench.handle.expect(":~")
- raw = main.ONOSbench.handle.before
- #if "NullLinkProvider" in raw and "links=14" in raw:
- if "links=14" in raw:
- break
- if i >= 9:
- main.log.error("Expected output not being recieved... continuing")
- main.log.info(raw)
- break
- time.sleep(2)
-
- if debug: main.log.debug("raw: " + raw)
-
- temp = raw.splitlines()
-
- if debug: main.log.debug("temp (after splitlines): " + str(temp))
-
- # Since the string is deterministic the date is always the 3rd element.
- # However, if the data were grepping for in the onos log changes then this will
- # not work. This is why we print out the raw and temp string so we can visually
- # check if everything is in the correct order. temp should like this:
- # temp = ['/onos$ onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep Top ',
- # 'ologyManager| tail -1', '2015-10-15 12:03:33,736 ... ]
- temp = temp[2]
-
- if debug: main.log.debug("temp (checking for date): " + str(temp))
-
- cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1]
-
- if debug: main.log.info("Cut timestamp: " + cutTimestamp)
-
- #validate link count and flow count
- for i in range(0, 40):
- main.ONOSbench.handle.sendline("onos $OC1 summary")
- main.ONOSbench.handle.expect(":~")
- linkCheck = main.ONOSbench.handle.before
- #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
- if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
- break
- if i == 39:
- main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
- main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"], "d")
-
- time.sleep(5) #trying to avoid negative values
-
- #intents events metrics installed timestamp
- IEMtimestamps = [0]*(clusterCount + 1)
- installedTemp = [0]*(clusterCount + 1)
- for node in range(1, clusterCount +1):
- cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
- raw = ""
- while "epoch)" not in raw:
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
- raw = main.ONOSbench.handle.before
-
- print(raw)
-
- intentsTimestamps = {}
- rawTimestamps = raw.splitlines()
- for line in rawTimestamps:
- if "Timestamp" in line and "grep" not in line:
- metricKey = (line.split(" "))[1]
- metricTimestamp = (line.split(" ")[len(line.split(" ")) -1]).replace("epoch)=","")
- metricTimestamp = float(metricTimestamp)
- metricTimestamp = numpy.divide(metricTimestamp, 1000)
- if debug: main.log.info(repr(metricTimestamp))
- intentsTimestamps[metricKey] = metricTimestamp
- if metricKey == "Installed":
- installedTemp[node] = metricTimestamp
-
- main.log.info("Node: " + str(node) + " Timestamps: " + str(intentsTimestamps))
- IEMtimestamps[node] = intentsTimestamps
-
- myMax = max(installedTemp)
- indexOfMax = installedTemp.index(myMax)
-
- #number crunch
- for metric in timestampMetrics: #this is where we sould add support for computing other timestamp metrics
- if metric == "Installed":
- if run >= warmUp:
- main.log.report("link cut timestamp: " + cutTimestamp)
- #readableInstalledTimestamp = str(intentsTimestamps["Installed"])
- readableInstalledTimestamp = str(myMax)
-
- #main.log.report("Intent Installed timestamp: " + str(intentsTimestamps["Installed"]))
- main.log.report("Intent Installed timestamp: " + str(myMax))
-
- cutEpoch = time.mktime(time.strptime(cutTimestamp, "%Y-%m-%d %H:%M:%S,%f"))
- if debug: main.log.info("cutEpoch=" + str(cutEpoch))
- #rerouteLatency = float(intentsTimestamps["Installed"] - cutEpoch)
- rerouteLatency = float(myMax - cutEpoch)
-
- rerouteLatency = numpy.divide(rerouteLatency, 1000)
- main.log.report("Reroute latency:" + str(rerouteLatency) + " (seconds)\n ")
- myResult[run-warmUp][0] = rerouteLatency
- myResult[run-warmUp][1] = indexOfMax
- if debug: main.log.info("Latency: " + str(myResult[run-warmUp][0]))
- if debug: main.log.info("last node: " + str(myResult[run-warmUp][1]))
-
- cmd = """ onos $OC1 null-link "null:0000000000000004/1 null:0000000000000003/2 up" """
- if debug: main.log.info(cmd)
- main.ONOSbench.handle.sendline(cmd)
- main.ONOSbench.handle.expect(":~")
-
- #wait for intent withdraw
- main.ONOSbench.handle.sendline(withdrawCmd)
- main.log.info(withdrawCmd)
- main.ONOSbench.handle.expect(":~")
- if debug: main.log.info(main.ONOSbench.handle.before)
- main.ONOSbench.handle.sendline("onos $OC1 intents|grep WITHDRAWN|wc -l")
- main.ONOSbench.handle.expect(":~")
- intentWithdrawCheck = main.ONOSbench.handle.before
- if (str(intents)) in intentWithdrawCheck:
- main.log.info("intents withdrawn")
- if debug: main.log.info(intentWithdrawCheck)
-
- # wait for links to be reestablished
- for i in range(0, 10):
- main.ONOSbench.handle.sendline("onos $OC1 summary")
- main.ONOSbench.handle.expect(":~")
- linkCheck = main.ONOSbench.handle.before
- if "links=16," in linkCheck:
- break
- time.sleep(1)
- if i == 9:
- main.log.info("Links Failed to reconnect, next iteration of data invalid." + linkCheck)
-
- if run < warmUp:
- main.log.info("Warm up run " + str(run+1) + " completed")
-
- if debug: main.log.info(myResult)
- latTemp = []
- nodeTemp = []
- for i in myResult:
- latTemp.append(i[0])
- nodeTemp.append(i[1])
-
- mode = {}
- for i in nodeTemp:
- if i in mode:
- mode[i] += 1
+ print( main.intentsList )
+ for batchSize in main.intentsList:
+ main.log.report("Intent Batch size: " + str(batchSize) + "\n ")
+ main.LatencyList = []
+ for run in range( 0, (main.warmUp + main.sampleSize) ):
+ if run >= main.warmUp:
+ main.log.info( "================================================" )
+ main.log.info( "Starting test iteration " + str(run - main.warmUp) )
+ main.log.info( "================================================" )
else:
- mode[i] = 1
+ main.log.info( "====================Warm Up=====================" )
- for i in mode.keys():
- if mode[i] == max(mode.values()):
- nodeMode = i
+ # push intents
+ main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
+ offset=1,options="-i",timeout=main.timeout)
- average = numpy.average(latTemp)
- stdDev = numpy.std(latTemp)
+ # check links and flows
+ k = 0
+ verify = main.FALSE
+ linkCheck = 0
+ flowsCheck = 0
+ while k <= main.verifyAttempts:
+ time.sleep( main.verifySleep )
+ summary = json.loads( main.CLIs[0].summary(timeout=main.timeout) )
+ linkCheck = summary.get("links")
+ flowsCheck = summary.get("flows")
+ if linkCheck == 16 and flowsCheck == batchSize*7:
+ main.log.info( "links: {}, flows: {} ".format(linkCheck,flowsCheck) )
+ verify = main.TRUE
+ break
+ k += 1
+ if not verify:
+ main.log.warn( "Links or flows number are not match!")
+ main.log.warn( "links: {}, flows: {} ".format(linkCheck, flowsCheck) )
+ continue
- average = numpy.multiply(average, 1000)
- stdDev = numpy.multiply(stdDev, 1000)
+ # Bring link down
+ main.CLIs[0].link("0000000000000004/1", "0000000000000003/2", "down",
+ timeout=main.timeout, showResponse=False )
+ verify = main.FALSE
+ k = 0
+ topoManagerLog = ""
+ while k <= main.verifyAttempts:
+ time.sleep(main.verifySleep)
+ summary = json.loads( main.CLIs[0].summary(timeout=main.timeout) )
+ linkCheck = summary.get("links")
+ flowsCheck = summary.get("flows")
+ if linkCheck == 14:
+ main.log.info( "links: {}, flows: {} ".format(linkCheck, flowsCheck) )
+ verify = main.TRUE
+ break
+ k += 1
+ if not verify:
+ main.log.warn( "Links number are not match in TopologyManager log!" )
+ main.log.warn( topoManagerLog )
+ continue
- main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents))
- main.log.report("Latency average:................" + str(average))
- main.log.report("Latency standard deviation:....." + str(stdDev))
- main.log.report("Mode of last node to respond:..." + str(nodeMode))
+ try:
+ # expect twice to clean the pexpect buffer
+ main.ONOSbench.handle.sendline("")
+ main.ONOSbench.handle.expect("\$")
+ main.ONOSbench.handle.expect("\$")
+ # send line by using bench, can't use driver because pexpect buffer problem
+ cmd = "onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep TopologyManager| tail -1"
+ main.ONOSbench.handle.sendline(cmd)
+ time.sleep(1)
+ main.ONOSbench.handle.expect(":~")
+ topoManagerLog = main.ONOSbench.handle.before
+ topoManagerLogTemp = topoManagerLog.splitlines()
+ # To make sure we get correct topology log
+ for lines in topoManagerLogTemp:
+ if "creationTime" in lines:
+ topoManagerLog = lines
+ main.log.info("Topology Manager log:")
+ print(topoManagerLog)
+ cutTimestamp = float(topoManagerLog.split("creationTime=")[1].split(",")[0])
+ except:
+ main.log.error("Topology Log is not correct!")
+ print(topoManagerLog)
+ # If we got wrong Topology log, we should skip this iteration, and continue for next one
+ continue
+
+ installedTemp = []
+ time.sleep(1)
+ for cli in main.CLIs:
+ tempJson = json.loads( cli.intentsEventsMetrics() )
+ Installedtime = tempJson.get('intentInstalledTimestamp').get('value')
+ installedTemp.append(float(Installedtime))
+ for i in range(0, len(installedTemp)):
+ main.log.info("ONOS Node {} Installed Time stemp: {}".format((i+1),installedTemp[i]))
+ maxInstallTime = float( max(installedTemp) )
+ if run >= main.warmUp:
+ main.log.info( "Installed time stemp: {0:f}".format( maxInstallTime ) )
+ main.log.info("CutTimestamp: {0:f}".format( cutTimestamp) )
+ # Both timeStemps are milliseconds
+ main.log.info( "Latency: {0:f}".format( float(maxInstallTime-cutTimestamp)) )
+ main.LatencyList.append(float(maxInstallTime-cutTimestamp))
+
+ # Verify Summary after we bring up link, and withdrawn intents
+ main.CLIs[0].link( "0000000000000004/1", "0000000000000003/2", "up",
+ timeout=main.timeout )
+ k = 0
+ verify = main.FALSE
+ linkCheck = 0
+ flowsCheck = 0
+ while k <= main.verifyAttempts:
+ time.sleep(main.verifySleep)
+ main.CLIs[0].removeAllIntents( purge=True, sync=True, timeout=main.timeout )
+ time.sleep(1)
+ main.CLIs[0].purgeWithdrawnIntents()
+ summary = json.loads( main.CLIs[0].summary() )
+ linkCheck = summary.get("links")
+ flowsCheck = summary.get("flows")
+ intentCheck = summary.get("intents")
+ if linkCheck == 16 and flowsCheck == 0 and intentCheck == 0:
+ main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
+ verify = main.TRUE
+ break
+ k += 1
+ if not verify:
+ main.log.error("links, flows, or intents are not correct!")
+ main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
+ continue
+
+ aveLatency=0
+ stdLatency=0
+ aveLatency = numpy.average(main.LatencyList)
+ stdLatency = numpy.std(main.LatencyList)
+ main.log.report("Scale: " + str(main.numCtrls) + " \tIntent batch: " + str(batchSize))
+ main.log.report("Latency average:................" + str(aveLatency))
+ main.log.report("Latency standard deviation:....." + str(stdLatency))
main.log.report("________________________________________________________")
- resultsDB = open("/tmp/IntentRerouteLatDBWithFlowObj", "a")
- resultsDB.write("'" + commit + "',")
- resultsDB.write(str(clusterCount) + ",")
- resultsDB.write(str(intents) + ",")
- resultsDB.write(str(average) + ",")
- resultsDB.write(str(stdDev) + "\n")
+ resultsDB = open(main.dbFileName, "a")
+ resultsDB.write("'" + main.commit + "',")
+ resultsDB.write(str(main.numCtrls) + ",")
+ resultsDB.write(str(batchSize) + ",")
+ resultsDB.write(str(aveLatency) + ",")
+ resultsDB.write(str(stdLatency) + "\n")
resultsDB.close()
-
- main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
+ del main.scale[0]