Minor bug fixes and refactoring to scale-out scenario
diff --git a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params
deleted file mode 100644
index b898149..0000000
--- a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params
+++ /dev/null
@@ -1,98 +0,0 @@
-<PARAMS>
- <testcases>1,2,3,4,2,3,4,2,3,4,2,3</testcases>
-
- <ENV>
- <cellName>topo_perf_test</cellName>
- <cellFeatures>"org.onosproject.metrics,org.onosproject.openflow"</cellFeatures>
- </ENV>
-
- <GIT>
- #autoPull 'on' or 'off'
- <autoPull>off</autoPull>
- <checkout>master</checkout>
- </GIT>
-
- <CTRL>
- <user>sdn</user>
- <ip1>10.254.1.201</ip1>
- <port1>6633</port1>
- <ip2>10.254.1.202</ip2>
- <port2>6633</port2>
- <ip3>10.254.1.203</ip3>
- <port3>6633</port3>
- <ip4>10.254.1.204</ip4>
-
- <ip5>10.254.1.205</ip5>
- <ip6>10.254.1.206</ip6>
- <ip7>10.254.1.207</ip7>
- </CTRL>
-
- <MN>
- <ip1>10.254.1.200</ip1>
- <ip2>10.254.1.200</ip2>
- </MN>
-
- <BENCH>
- <ip>10.254.1.200</ip>
- </BENCH>
-
- <TSHARK>
- # What to 'grep' for tshark outputs
-
- <ofpPortStatus>OF 1.3 146</ofpPortStatus>
- <ofpRoleReply>OF 1.3 90 of_role_reply</ofpRoleReply>
- <featureReply>OF 1.3 98 of_features_reply</featureReply>
- <roleRequest>OF 1.3 90 of_role_request</roleRequest>
- <tcpSynAck>TCP 74 6633</tcpSynAck>
- <finAckSequence>FIN</finAckSequence>
- </TSHARK>
-
- <TEST>
- #'on' or 'off' debug mode.
- #If on, logging will be more verbose and
- #tshark pcap will be enabled
- #pcap file located at /tmp/'capture_name'
- <debugMode>on</debugMode>
- <onosLogFile>/opt/onos/log/karaf*</onosLogFile>
- <mci>off</mci>
-
- <topoConfigFile>
- single_topo_event_accumulator.cfg
- </topoConfigFile>
- <topoConfigName>
- org.onlab.onos.net.topology.impl.DefaultTopologyProvider.cfg
- </topoConfigName>
-
- #Number of times to iterate each case
- <numIter>25</numIter>
- <numSwitch>2</numSwitch>
- #Number of iterations to ignore initially
- <iterIgnore>2</iterIgnore>
-
- <singleSwThreshold>0.0,1000</singleSwThreshold>
- <portUpThreshold>0.0,1000</portUpThreshold>
- <portDownThreshold>0.0,1000</portDownThreshold>
- <linkUpThreshold>0.0,10000</linkUpThreshold>
- <linkDownThreshold>0.0,10000</linkDownThreshold>
- <swDisc100Threshold>0.0,10000</swDisc100Threshold>
-
- <tabletFile>tablets_3node.json</tabletFile>
- </TEST>
-
- <DB>
- <postToDB>on</postToDB>
- <portEventResultPath>
- /home/admin/ONLabTest/TestON/tests/TopoPerfNextBM/portEventResultDb.log
- </portEventResultPath>
- <switchEventResultPath>
- /home/admin/ONLabTest/TestON/tests/TopoPerfNextBM/switchEventResultDb.log
- </switchEventResultPath>
- </DB>
-
- <JSON>
- <deviceTimestamp>topologyDeviceEventTimestamp</deviceTimestamp>
- <hostTimestamp>topologyHostEventTimestamp</hostTimestamp>
- <linkTimestamp>topologyLinkEventTimestamp</linkTimestamp>
- <graphTimestamp>topologyGraphEventTimestamp</graphTimestamp>
- </JSON>
-</PARAMS>
diff --git a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py
index de0ad05..e73c588 100644
--- a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py
+++ b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py
@@ -119,7 +119,7 @@
CLIs[0].activateApp( 'org.onosproject.openflow' )
main.step( 'Configuring application parameters' )
-<<<<<<< HEAD
+
configName = 'org.onosproject.net.topology.impl.DefaultTopologyProvider'
configParam = 'maxEvents 1'
main.ONOSbench.onosCfgSet( nodeIpList[0], configName, configParam )
@@ -128,28 +128,6 @@
configParam = 'maxIdleMs 0'
main.ONOSbench.onosCfgSet( nodeIpList[0], configName, configParam )
-=======
- # TODO: Check driver for this functionality
- main.ONOSbench.handle.sendline(
- 'onos '+nodeIpList[0]+
- ' cfg set org.onosproject.net.'+
- 'topology.impl.DefaultTopologyProvider'+
- ' maxEvents 1')
- main.ONOSbench.handle.expect(":~")
- main.ONOSbench.handle.sendline(
- 'onos '+nodeIpList[0]+
- ' cfg set org.onosproject.net.'+
- 'topology.impl.DefaultTopologyProvider'+
- ' maxBatchMs 0')
- main.ONOSbench.handle.expect(":~")
- main.ONOSbench.handle.sendline(
- 'onos '+nodeIpList[0]+
- ' cfg set org.onosproject.net.'+
- 'topology.impl.DefaultTopologyProvider'+
- ' maxIdleMs 0')
- main.ONOSbench.handle.expect(":~")
-
->>>>>>> c16e983dcc6a370bcdebdf4168770cee5c619a7d
utilities.assert_equals(expect=main.TRUE,
actual=cellFileResult and cellApplyResult and\
verifyCellResult and checkoutResult and\
@@ -388,7 +366,6 @@
if jsonObj:
graphTimestamp = jsonObj[graphTimestampKey]['value']
deviceTimestamp = jsonObj[deviceTimestampKey]['value']
-<<<<<<< HEAD
else:
main.log.error( "Unexpected JSON object" )
# If we could not obtain the JSON object,
@@ -414,33 +391,6 @@
main.log.info("ONOS "+str(nodeNum)+ " end-to-end: "+
str(endToEnd) + " ms")
else:
-=======
- else:
- main.log.error( "Unexpected JSON object" )
- # If we could not obtain the JSON object,
- # set the timestamps to 0, which will be
- # excluded from the measurement later on
- # (realized as invalid)
- graphTimestamp = 0
- deviceTimestamp = 0
-
- endToEnd = int(graphTimestamp) - int(t0Tcp)
-
- # Below are measurement breakdowns of the end-to-end
- # measurement.
- tcpToFeature = int(featureTimestamp) - int(t0Tcp)
- featureToRole = int(roleTimestamp) - int(featureTimestamp)
- roleToOfp = int(t0Ofp) - int(roleTimestamp)
- ofpToDevice = int(deviceTimestamp) - int(t0Ofp)
- deviceToGraph = float(graphTimestamp) - float(deviceTimestamp)
-
- if endToEnd > thresholdMin and\
- endToEnd < thresholdMax and i >= iterIgnore:
- endToEndLatNodeIter[node][i] = endToEnd
- main.log.info("ONOS "+str(nodeNum)+ " end-to-end: "+
- str(endToEnd) + " ms")
- else:
->>>>>>> c16e983dcc6a370bcdebdf4168770cee5c619a7d
main.log.info("ONOS "+str(nodeNum)+ " end-to-end "+
"measurement ignored due to excess in "+
"threshold or premature iteration")
@@ -491,7 +441,7 @@
main.log.info("ONOS "+str(nodeNum)+ " device-to-graph: "+
str(deviceToGraph) + " ms")
else:
- main.log.info("ONOS "+str(nodeNum)+ " device=to-graph "+
+ main.log.info("ONOS "+str(nodeNum)+ " device-to-graph "+
"measurement ignored due to excess in "+
"threshold or premature iteration")
@@ -508,31 +458,37 @@
for device in deviceId:
deviceList.append(device['id'])
- # TODO: Measure switch down metrics
+ # Measure switch down metrics
# TCP FIN/ACK -> TCP FIN
# TCP FIN -> Device Event
# Device Event -> Graph Event
-<<<<<<< HEAD
-
# Capture switch down FIN / ACK packets
main.ONOS1.tsharkGrep( tsharkFinAckSequence, tsharkFinAckOutput,
grepOptions = '-A 1' )
time.sleep( 5 )
-=======
->>>>>>> c16e983dcc6a370bcdebdf4168770cee5c619a7d
+
+ removeJsonList = []
main.step('Remove switch from controller')
main.Mininet1.deleteSwController('s3')
- #firstDevice = deviceList[0]
- firstDevice = "of:0000000000000003"
+ firstDevice = deviceList[0]
main.log.info( "Removing device " +str(firstDevice)+
" from ONOS" )
+
+ time.sleep( 5 )
+
+ # We need to get metrics before removing
+ # device from the store below.
+ for node in range(0, clusterCount):
+ metricsSwDown = CLIs[node].topologyEventsMetrics
+ jsonStr = metricsSwDown()
+ removeJsonList.append( json.loads(jsonStr) )
+
#if deviceId:
main.ONOS1cli.deviceRemove(firstDevice)
- time.sleep( 5 )
main.ONOS1.tsharkStop()
main.log.info('Copying over tshark files')
@@ -553,10 +509,8 @@
if len(obj) > 1:
if index == 1:
tFinAck = float(obj[1]) * 1000.0
- print "*** fin ack time :"+str(tFinAck)
elif index == 3:
tAck = float(obj[1]) * 1000.0
- print "*** Ack time: "+str(tAck)
else:
main.log.error('Tshark output file for OFP' +
' returned unexpected results')
@@ -572,9 +526,7 @@
for node in range(0, clusterCount):
nodeNum = node+1
- metricsSwDown = CLIs[node].topologyEventsMetrics
- jsonStr = metricsSwDown()
- jsonObj = json.loads(jsonStr)
+ jsonObj = removeJsonList[node]
if jsonObj:
graphTimestamp = jsonObj[graphTimestampKey]['value']
deviceTimestamp = jsonObj[deviceTimestampKey]['value']
@@ -641,28 +593,21 @@
roleToOfpList = []
ofpToDeviceList = []
deviceToGraphList = []
-<<<<<<< HEAD
+
finAckTransactionList = []
ackToDeviceList = []
deviceToGraphDiscList = []
-=======
->>>>>>> c16e983dcc6a370bcdebdf4168770cee5c619a7d
# LatNodeIter 2d arrays contain all iteration latency
# for each node of the current scale cluster size
-
-<<<<<<< HEAD
# Switch connection measurements
-=======
- # TODO: Use the new breakdown latency lists
-
->>>>>>> c16e983dcc6a370bcdebdf4168770cee5c619a7d
+ # Set further acceptance criteria for measurements
+ # here if you would like to filter reporting results
for item in endToEndLatNodeIter[node]:
if item > 0.0:
endToEndList.append(item)
for item in tcpToFeatureLatNodeIter[node]:
-<<<<<<< HEAD
if item > 0.0:
tcpToFeatureList.append(item)
@@ -675,16 +620,13 @@
roleToOfpList.append(item)
for item in roleReplyToDeviceLatNodeIter[node]:
-=======
if item > 0.0:
tcpToFeatureList.append(item)
for item in featureToRoleRequestLatNodeIter[node]:
->>>>>>> c16e983dcc6a370bcdebdf4168770cee5c619a7d
if item > 0.0:
featureToRoleList.append(item)
-<<<<<<< HEAD
for item in deviceToGraphLatNodeIter[node]:
if item > 0.0:
deviceToGraphList.append(item)
@@ -701,20 +643,7 @@
for item in deviceToGraphDiscLatNodeIter[node]:
if item > 0.0:
deviceToGraphDiscList.append(item)
-=======
- for item in roleRequestToRoleReplyLatNodeIter[node]:
- if item > 0.0:
- roleToOfpList.append(item)
-
- for item in roleReplyToDeviceLatNodeIter[node]:
- if item > 0.0:
- ofpToDeviceList.append(item)
-
- for item in deviceToGraphLatNodeIter[node]:
- if item > 0.0:
- deviceToGraphList.append(item)
->>>>>>> c16e983dcc6a370bcdebdf4168770cee5c619a7d
-
+
endToEndAvg = round(numpy.mean(endToEndList), 2)
endToEndStdDev = round(numpy.std(endToEndList), 2)
@@ -733,7 +662,6 @@
deviceToGraphAvg = round(numpy.mean(deviceToGraphList), 2)
deviceToGraphStdDev = round(numpy.std(deviceToGraphList), 2)
-<<<<<<< HEAD
finAckAvg = round(numpy.mean(finAckTransactionList), 2)
finAckStdDev = round(numpy.std(finAckTransactionList), 2)
@@ -743,8 +671,6 @@
deviceToGraphDiscAvg = round(numpy.mean(deviceToGraphDiscList), 2)
deviceToGraphDiscStdDev = round(numpy.std(deviceToGraphDiscList), 2)
-=======
->>>>>>> c16e983dcc6a370bcdebdf4168770cee5c619a7d
main.log.report(' - Node ' + str(node + 1) + ' Summary - ')
main.log.report(' - Switch Connection Statistics - ')
@@ -777,7 +703,6 @@
main.log.report( 'Device-to-graph Std dev: ' +
str(deviceToGraphStdDev) + ' ms')
-<<<<<<< HEAD
main.log.report(' - Switch Disconnection Statistics - ')
main.log.report(' Fin/Ack-to-Ack Avg: ' + str(finAckAvg) + ' ms')
main.log.report(' Fin/Ack-to-Ack Std dev: ' +
@@ -793,8 +718,6 @@
main.log.report(' Device-to-graph (disconnect) Std dev: ' +
str(deviceToGraphDiscStdDev) + ' ms')
-=======
->>>>>>> c16e983dcc6a370bcdebdf4168770cee5c619a7d
dbCmdList.append(
"INSERT INTO switch_latency_tests VALUES('" +
timeToPost + "','switch_latency_results'," +
@@ -840,9 +763,10 @@
assertion = main.TRUE
numIter = main.params['TEST']['numIter']
iterIgnore = int(main.params['TEST']['iterIgnore'])
- deviceTimestamp = main.params['JSON']['deviceTimestamp']
- graphTimestamp = main.params['JSON']['graphTimestamp']
- linkTimestamp = main.params['JSON']['linkTimestamp']
+
+ deviceTimestampKey = main.params['JSON']['deviceTimestamp']
+ graphTimestampKey = main.params['JSON']['graphTimestamp']
+ linkTimestampKey = main.params['JSON']['linkTimestamp']
tsharkPortUp = '/tmp/tshark_port_up.txt'
tsharkPortDown = '/tmp/tshark_port_down.txt'
@@ -898,22 +822,34 @@
for i in range(0, int(numIter)):
main.step('Starting wireshark capture for port status down')
main.ONOS1.tsharkGrep(tsharkPortStatus, tsharkPortDown)
+
time.sleep(5)
+
main.step('Disable port: ' + interfaceConfig)
main.Mininet1.handle.sendline('sh ifconfig ' +
interfaceConfig + ' down')
main.Mininet1.handle.expect('mininet>')
+
time.sleep(3)
+
main.ONOS1.tsharkStop()
+
os.system('scp ' + ONOSUser + '@' + ONOS1Ip + ':' +
tsharkPortDown + ' /tmp/')
+
fPortDown = open(tsharkPortDown, 'r')
fLine = fPortDown.readline()
objDown = fLine.split(' ')
if len(fLine) > 0:
timestampBeginPtDown = int(float(objDown[1]) * 1000)
+ # At times, tshark reports timestamp at the 3rd
+ # index of the array. If initial readings were
+ # unlike the epoch timestamp, then check the 3rd
+ # index and set that as a timestamp
if timestampBeginPtDown < 1400000000000:
timestampBeginPtDown = int(float(objDown[2]) * 1000)
+ # If there are any suspicion of invalid results
+ # check this reported value
main.log.info('Port down begin timestamp: ' +
str(timestampBeginPtDown))
else:
@@ -929,9 +865,9 @@
jsonObj = json.loads(jsonStrDown)
if jsonObj:
- graphTimestamp = jsonObj[graphTimestamp]['value']
- deviceTimestamp = jsonObj[deviceTimestamp]['value']
- linkTimestamp = jsonObj[linkTimestamp]['value']
+ graphTimestamp = jsonObj[graphTimestampKey]['value']
+ deviceTimestamp = jsonObj[deviceTimestampKey]['value']
+ linkTimestamp = jsonObj[linkTimestampKey]['value']
else:
main.log.error( "Unexpected json object" )
graphTimestamp = 0
@@ -943,7 +879,7 @@
ptDownLinkToOfp = int(linkTimestamp) - int(timestampBeginPtDown)
if ptDownGraphToOfp > downThresholdMin and\
- ptDownGraphToOfp < downThresholdMax and i > iterIgnore:
+ ptDownGraphToOfp < downThresholdMax and i >= iterIgnore:
portDownGraphNodeIter[node][i] = ptDownGraphToOfp
main.log.info("ONOS "+str(nodeNum)+
" port down graph-to-ofp: "+
@@ -954,8 +890,8 @@
" due to excess in threshold or premature iteration")
if ptDownDeviceToOfp > downThresholdMin and\
- ptDownDeviceToOfp < downThresholdMax and i > iterIgnore:
- portDownDeviceNodeIter[node][i] = ptDownDeviceToOfp
+ ptDownDeviceToOfp < downThresholdMax and i >= iterIgnore:
+ portDownDevNodeIter[node][i] = ptDownDeviceToOfp
main.log.info("ONOS "+str(nodeNum)+
" port down device-to-ofp: "+
str(ptDownDeviceToOfp) + " ms")
@@ -965,7 +901,7 @@
" due to excess in threshold or premature iteration")
if ptDownLinkToOfp > downThresholdMin and\
- ptDownLinkToOfp < downThresholdMax and i > iterIgnore:
+ ptDownLinkToOfp < downThresholdMax and i >= iterIgnore:
portDownLinkNodeIter[node][i] = ptDownLinkToOfp
main.log.info("ONOS "+str(nodeNum)+
" port down link-to-ofp: "+
@@ -1012,9 +948,9 @@
jsonObj = json.loads(jsonStrUp)
if jsonObj:
- graphTimestamp = jsonObj[graphTimestamp]['value']
- deviceTimestamp = jsonObj[deviceTimestamp]['value']
- linkTimestamp = jsonObj[linkTimestamp]['value']
+ graphTimestamp = jsonObj[graphTimestampKey]['value']
+ deviceTimestamp = jsonObj[deviceTimestampKey]['value']
+ linkTimestamp = jsonObj[linkTimestampKey]['value']
else:
main.log.error( "Unexpected json object" )
graphTimestamp = 0
@@ -1038,7 +974,7 @@
if ptUpDeviceToOfp > upThresholdMin and\
ptUpDeviceToOfp < upThresholdMax and i > iterIgnore:
- portUpDeviceNodeIter[node][i] = ptUpDeviceToOfp
+ portUpDevNodeIter[node][i] = ptUpDeviceToOfp
main.log.info("ONOS "+str(nodeNum)+
" port up device-to-ofp: "+
str(ptUpDeviceToOfp) + " ms")
@@ -1136,6 +1072,9 @@
fResult.write(line + '\n')
fResult.close()
+
+ # Delete switches from controller to prepare for next
+ # set of tests
main.Mininet1.deleteSwController('s1')
main.Mininet1.deleteSwController('s2')
utilities.assert_equals(expect=main.TRUE,
@@ -1208,10 +1147,18 @@
if not isup:
main.log.error( "ONOS "+str(node) + " did not start" )
- for node in range( 1, clusterCount + 1):
- CLIs[node-1].startOnosCli
-
- main.step( 'activating essential applications' )
- CLIs[node-1].activateApp( 'org.onosproject.metrics' )
- CLIs[node-1].activateApp( 'org.onosproject.openflow' )
+ for node in range( 0, clusterCount ):
+ CLIs[node].startOnosCli( cellIp[node] )
+
+ main.step( 'Setting configurations for metrics' )
+ configParam = 'maxEvents 1'
+ main.ONOSbench.onosCfgSet( nodeIpList[0], configName, configParam )
+ configParam = 'maxBatchMs 0'
+ main.ONOSbench.onosCfgSet( nodeIpList[0], configName, configParam )
+ configParam = 'maxIdleMs 0'
+ main.ONOSbench.onosCfgSet( nodeIpList[0], configName, configParam )
+
+ main.step( 'Activating essential applications' )
+ CLIs[0].activateApp( 'org.onosproject.metrics' )
+ CLIs[0].activateApp( 'org.onosproject.openflow' )