blob: 12e53e5b2d47c7aadb15e1a3cb521afa1440fb8d [file] [log] [blame]
# TopoPerfNext
#
# Topology Performance test for ONOS-next
#
# andrew@onlab.us
#
# If your machine does not come with numpy
# run the following command:
# sudo apt-get install python-numpy python-scipy
import time
import sys
import os
import re
class TopoPerfNext:
def __init__( self ):
self.default = ''
def CASE1( self, main ):
"""
ONOS startup sequence
"""
import time
# Global cluster count for scale-out purposes
global clusterCount
# Set initial cluster count
clusterCount = 1
##
cellName = main.params[ 'ENV' ][ 'cellName' ]
gitPull = main.params[ 'GIT' ][ 'autoPull' ]
checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
MN1Ip = main.params[ 'MN' ][ 'ip1' ]
BENCHIp = main.params[ 'BENCH' ][ 'ip' ]
topoCfgFile = main.params[ 'TEST' ][ 'topoConfigFile' ]
topoCfgName = main.params[ 'TEST' ][ 'topoConfigName' ]
mvnCleanInstall = main.params[ 'TEST' ][ 'mci' ]
main.case( "Setting up test environment" )
main.log.info( "Copying topology event accumulator config" +
" to ONOS /package/etc" )
main.ONOSbench.handle.sendline( "cp ~/" +
topoCfgFile +
" ~/ONOS/tools/package/etc/" +
topoCfgName )
main.ONOSbench.handle.expect( "\$" )
main.log.report( "Setting up test environment" )
main.step( "Cleaning previously installed ONOS if any" )
main.ONOSbench.onosUninstall( nodeIp=ONOS2Ip )
main.ONOSbench.onosUninstall( nodeIp=ONOS3Ip )
main.ONOSbench.onosUninstall( nodeIp=ONOS4Ip )
main.ONOSbench.onosUninstall( nodeIp=ONOS5Ip )
main.ONOSbench.onosUninstall( nodeIp=ONOS6Ip )
#main.ONOSbench.onosUninstall( nodeIp=ONOS7Ip )
main.step( "Creating cell file" )
cellFileResult = main.ONOSbench.createCellFile(
BENCHIp, cellName, MN1Ip,
"onos-core,onos-app-metrics,onos-app-gui",
ONOS1Ip )
main.step( "Applying cell file to environment" )
cellApplyResult = main.ONOSbench.setCell( cellName )
verifyCellResult = main.ONOSbench.verifyCell()
# NOTE: This step may be removed after proper
# copy cat log functionality
main.step( "Removing raft/copy-cat logs from ONOS nodes" )
main.ONOSbench.onosRemoveRaftLogs()
time.sleep( 30 )
main.step( "Git checkout and pull " + checkoutBranch )
if gitPull == 'on':
# checkoutResult = \
# main.ONOSbench.gitCheckout( checkoutBranch )
checkoutResult = main.TRUE
pullResult = main.ONOSbench.gitPull()
else:
checkoutResult = main.TRUE
pullResult = main.TRUE
main.log.info( "Skipped git checkout and pull" )
main.log.report( "Commit information - " )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean & install" )
if mvnCleanInstall == 'on':
mvnResult = main.ONOSbench.cleanInstall()
elif mvnCleanInstall == 'off':
main.log.info("mci turned off by settings")
mvnResult = main.TRUE
main.step( "Set cell for ONOS cli env" )
main.ONOS1cli.setCell( cellName )
# main.ONOS2cli.setCell( cellName )
# main.ONOS3cli.setCell( cellName )
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
main.step( "Installing ONOS package" )
install1Result = main.ONOSbench.onosInstall( node=ONOS1Ip )
#install2Result = main.ONOSbench.onosInstall( node=ONOS2Ip )
#install3Result = main.ONOSbench.onosInstall( node=ONOS3Ip )
time.sleep( 10 )
main.step( "Start onos cli" )
cli1 = main.ONOS1cli.startOnosCli( ONOS1Ip )
#cli2 = main.ONOS2cli.startOnosCli( ONOS2Ip )
#cli3 = main.ONOS3cli.startOnosCli( ONOS3Ip )
utilities.assert_equals( expect=main.TRUE,
actual=cellFileResult and cellApplyResult and
verifyCellResult and checkoutResult and
pullResult and mvnResult and
install1Result, # and install2Result and
# install3Result,
onpass="Test Environment setup successful",
onfail="Failed to setup test environment" )
def CASE2( self, main ):
"""
Assign s1 to ONOS1 and measure latency
There are 4 levels of latency measurements to this test:
1 ) End-to-end measurement: Complete end-to-end measurement
from TCP ( SYN/ACK ) handshake to Graph change
2 ) OFP-to-graph measurement: 'ONOS processing' snippet of
measurement from OFP Vendor message to Graph change
3 ) OFP-to-device measurement: 'ONOS processing without
graph change' snippet of measurement from OFP vendor
message to Device change timestamp
4 ) T0-to-device measurement: Measurement that includes
the switch handshake to devices timestamp without
the graph view change. ( TCP handshake -> Device
change )
"""
import time
import subprocess
import json
import requests
import os
import numpy
global clusterCount
ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
ONOSUser = main.params[ 'CTRL' ][ 'user' ]
defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
# Number of iterations of case
numIter = main.params[ 'TEST' ][ 'numIter' ]
# Number of first 'x' iterations to ignore:
iterIgnore = int( main.params[ 'TEST' ][ 'iterIgnore' ] )
# Timestamp 'keys' for json metrics output.
# These are subject to change, hence moved into params
deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
debugMode = main.params[ 'TEST' ][ 'debugMode' ]
onosLog = main.params[ 'TEST' ][ 'onosLogFile' ]
# Threshold for the test
thresholdStr = main.params[ 'TEST' ][ 'singleSwThreshold' ]
thresholdObj = thresholdStr.split( "," )
thresholdMin = int( thresholdObj[ 0 ] )
thresholdMax = int( thresholdObj[ 1 ] )
# List of switch add latency collected from
# all iterations
latencyEndToEndList = []
latencyOfpToGraphList = []
latencyOfpToDeviceList = []
latencyT0ToDeviceList = []
latencyTcpToOfpList = []
# Directory/file to store tshark results
tsharkOfOutput = "/tmp/tshark_of_topo.txt"
tsharkTcpOutput = "/tmp/tshark_tcp_topo.txt"
# String to grep in tshark output
tsharkTcpString = "TCP 74 " + defaultSwPort
tsharkOfString = "OFP 86 Vendor"
# Initialize assertion to TRUE
assertion = main.TRUE
localTime = time.strftime( '%x %X' )
localTime = localTime.replace( "/", "" )
localTime = localTime.replace( " ", "_" )
localTime = localTime.replace( ":", "" )
if debugMode == 'on':
main.ONOS1.tsharkPcap( "eth0",
"/tmp/single_sw_lat_pcap_" + localTime )
main.log.info( "Debug mode is on" )
main.log.report( "Latency of adding one switch to controller" )
main.log.report( "First " + str( iterIgnore ) + " iterations ignored" +
" for jvm warmup time" )
main.log.report( "Total iterations of test: " + str( numIter ) )
for i in range( 0, int( numIter ) ):
main.log.info( "Starting tshark capture" )
#* TCP [ ACK, SYN ] is used as t0A, the
# very first "exchange" between ONOS and
# the switch for end-to-end measurement
#* OFP [ Stats Reply ] is used for t0B
# the very last OFP message between ONOS
# and the switch for ONOS measurement
main.ONOS1.tsharkGrep( tsharkTcpString,
tsharkTcpOutput )
main.ONOS1.tsharkGrep( tsharkOfString,
tsharkOfOutput )
# Wait and ensure tshark is started and
# capturing
time.sleep( 10 )
main.log.info( "Assigning s1 to controller" )
main.Mininet1.assignSwController(
sw="1",
ip1=ONOS1Ip,
port1=defaultSwPort )
# Wait and ensure switch is assigned
# before stopping tshark
time.sleep( 30 )
main.log.info( "Stopping all Tshark processes" )
main.ONOS1.stopTshark()
# tshark output is saved in ONOS. Use subprocess
# to copy over files to TestON for parsing
main.log.info( "Copying over tshark files" )
# TCP CAPTURE ****
# Copy the tshark output from ONOS machine to
# TestON machine in tsharkTcpOutput directory>file
os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
tsharkTcpOutput + " /tmp/" )
tcpFile = open( tsharkTcpOutput, 'r' )
tempText = tcpFile.readline()
tempText = tempText.split( " " )
main.log.info( "Object read in from TCP capture: " +
str( tempText ) )
if len( tempText ) > 1:
t0Tcp = float( tempText[ 1 ] ) * 1000.0
else:
main.log.error( "Tshark output file for TCP" +
" returned unexpected results" )
t0Tcp = 0
assertion = main.FALSE
tcpFile.close()
#****************
# OF CAPTURE ****
os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
tsharkOfOutput + " /tmp/" )
ofFile = open( tsharkOfOutput, 'r' )
lineOfp = ""
# Read until last line of file
while True:
tempText = ofFile.readline()
if tempText != '':
lineOfp = tempText
else:
break
obj = lineOfp.split( " " )
main.log.info( "Object read in from OFP capture: " +
str( lineOfp ) )
if len( lineOfp ) > 1:
t0Ofp = float( obj[ 1 ] ) * 1000.0
else:
main.log.error( "Tshark output file for OFP" +
" returned unexpected results" )
t0Ofp = 0
assertion = main.FALSE
ofFile.close()
#****************
jsonStr1 = main.ONOS1cli.topologyEventsMetrics()
# Initialize scale-out variables
jsonStr2 = ""
jsonStr3 = ""
jsonStr4 = ""
jsonStr5 = ""
jsonStr6 = ""
jsonStr7 = ""
jsonObj1 = json.loads( jsonStr1 )
# Initialize scale-out variables
jsonObj2 = ""
jsonObj3 = ""
jsonObj4 = ""
jsonObj5 = ""
jsonObj6 = ""
jsonObj7 = ""
# Obtain graph timestamp. This timestsamp captures
# the epoch time at which the topology graph was updated.
graphTimestamp1 = \
jsonObj1[ graphTimestamp ][ 'value' ]
# Obtain device timestamp. This timestamp captures
# the epoch time at which the device event happened
deviceTimestamp1 = \
jsonObj1[ deviceTimestamp ][ 'value' ]
# t0 to device processing latency
deltaDevice1 = int( deviceTimestamp1 ) - int( t0Tcp )
# t0 to graph processing latency ( end-to-end )
deltaGraph1 = int( graphTimestamp1 ) - int( t0Tcp )
# ofp to graph processing latency ( ONOS processing )
deltaOfpGraph1 = int( graphTimestamp1 ) - int( t0Ofp )
# ofp to device processing latency ( ONOS processing )
deltaOfpDevice1 = float( deviceTimestamp1 ) - float( t0Ofp )
# TODO: Create even cluster number events
# Include scale-out measurements when applicable
if clusterCount >= 3:
jsonStr2 = main.ONOS2cli.topologyEventsMetrics()
jsonStr3 = main.ONOS3cli.topologyEventsMetrics()
jsonObj2 = json.loads( jsonStr2 )
jsonObj3 = json.loads( jsonStr3 )
graphTimestamp2 = \
jsonObj2[ graphTimestamp ][ 'value' ]
graphTimestamp3 = \
jsonObj3[ graphTimestamp ][ 'value' ]
deviceTimestamp2 = \
jsonObj2[ deviceTimestamp ][ 'value' ]
deviceTimestamp3 = \
jsonObj3[ deviceTimestamp ][ 'value' ]
deltaDevice2 = int( deviceTimestamp2 ) - int( t0Tcp )
deltaDevice3 = int( deviceTimestamp3 ) - int( t0Tcp )
deltaGraph2 = int( graphTimestamp2 ) - int( t0Tcp )
deltaGraph3 = int( graphTimestamp3 ) - int( t0Tcp )
deltaOfpGraph2 = int( graphTimestamp2 ) - int( t0Ofp )
deltaOfpGraph3 = int( graphTimestamp3 ) - int( t0Ofp )
deltaOfpDevice2 = float( deviceTimestamp2 ) -\
float( t0Ofp )
deltaOfpDevice3 = float( deviceTimestamp3 ) -\
float( t0Ofp )
else:
deltaDevice2 = 0
deltaDevice3 = 0
deltaGraph2 = 0
deltaGraph3 = 0
deltaOfpGraph2 = 0
deltaOfpGraph3 = 0
deltaOfpDevice2 = 0
deltaOfpDevice3 = 0
if clusterCount >= 5:
jsonStr4 = main.ONOS4cli.topologyEventsMetrics()
jsonStr5 = main.ONOS5cli.topologyEventsMetrics()
jsonObj4 = json.loads( jsonStr4 )
jsonObj5 = json.loads( jsonStr5 )
graphTimestamp4 = \
jsonObj4[ graphTimestamp ][ 'value' ]
graphTimestamp5 = \
jsonObj5[ graphTimestamp ][ 'value' ]
deviceTimestamp4 = \
jsonObj4[ deviceTimestamp ][ 'value' ]
deviceTimestamp5 = \
jsonObj5[ deviceTimestamp ][ 'value' ]
deltaDevice4 = int( deviceTimestamp4 ) - int( t0Tcp )
deltaDevice5 = int( deviceTimestamp5 ) - int( t0Tcp )
deltaGraph4 = int( graphTimestamp4 ) - int( t0Tcp )
deltaGraph5 = int( graphTimestamp5 ) - int( t0Tcp )
deltaOfpGraph4 = int( graphTimestamp4 ) - int( t0Ofp )
deltaOfpGraph5 = int( graphTimestamp5 ) - int( t0Ofp )
deltaOfpDevice4 = float( deviceTimestamp4 ) -\
float( t0Ofp )
deltaOfpDevice5 = float( deviceTimestamp5 ) -\
float( t0Ofp )
else:
deltaDevice4 = 0
deltaDevice5 = 0
deltaGraph4 = 0
deltaGraph5 = 0
deltaOfpGraph4 = 0
deltaOfpGraph5 = 0
deltaOfpDevice4 = 0
deltaOfpDevice5 = 0
if clusterCount >= 7:
jsonStr6 = main.ONOS6cli.topologyEventsMetrics()
jsonStr7 = main.ONOS7cli.topologyEventsMetrics()
jsonObj6 = json.loads( jsonStr6 )
jsonObj7 = json.loads( jsonStr7 )
graphTimestamp6 = \
jsonObj6[ graphTimestamp ][ 'value' ]
graphTimestamp7 = \
jsonObj7[ graphTimestamp ][ 'value' ]
deviceTimestamp6 = \
jsonObj6[ deviceTimestamp ][ 'value' ]
deviceTimestamp7 = \
jsonObj7[ deviceTimestamp ][ 'value' ]
deltaDevice6 = int( deviceTimestamp6 ) - int( t0Tcp )
deltaDevice7 = int( deviceTimestamp7 ) - int( t0Tcp )
deltaGraph6 = int( graphTimestamp6 ) - int( t0Tcp )
deltaGraph7 = int( graphTimestamp7 ) - int( t0Tcp )
deltaOfpGraph6 = int( graphTimestamp6 ) - int( t0Ofp )
deltaOfpGraph7 = int( graphTimestamp7 ) - int( t0Ofp )
deltaOfpDevice6 = float( deviceTimestamp6 ) -\
float( t0Ofp )
deltaOfpDevice7 = float( deviceTimestamp7 ) -\
float( t0Ofp )
else:
deltaDevice6 = 0
deltaDevice7 = 0
deltaGraph6 = 0
deltaGraph7 = 0
deltaOfpGraph6 = 0
deltaOfpGraph7 = 0
deltaOfpDevice6 = 0
deltaOfpDevice7 = 0
# Get average of delta from all instances
avgDeltaDevice = \
( int( deltaDevice1 ) +
int( deltaDevice2 ) +
int( deltaDevice3 ) +
int( deltaDevice4 ) +
int( deltaDevice5 ) +
int( deltaDevice6 ) +
int( deltaDevice7 ) ) / clusterCount
# Ensure avg delta meets the threshold before appending
if avgDeltaDevice > 0.0 and avgDeltaDevice < 10000\
and int( i ) > iterIgnore:
latencyT0ToDeviceList.append( avgDeltaDevice )
else:
main.log.info(
"Results for t0-to-device ignored" +
"due to excess in threshold / warmup iteration." )
# Get average of delta from all instances
# TODO: use max delta graph
#maxDeltaGraph = max( three )
avgDeltaGraph = \
( int( deltaGraph1 ) +
int( deltaGraph2 ) +
int( deltaGraph3 ) +
int( deltaGraph4 ) +
int( deltaGraph5 ) +
int( deltaGraph6 ) +
int( deltaGraph7 ) ) / clusterCount
# Ensure avg delta meets the threshold before appending
if avgDeltaGraph > 0.0 and avgDeltaGraph < 10000\
and int( i ) > iterIgnore:
latencyEndToEndList.append( avgDeltaGraph )
else:
main.log.info( "Results for end-to-end ignored" +
"due to excess in threshold" )
avgDeltaOfpGraph = \
( int( deltaOfpGraph1 ) +
int( deltaOfpGraph2 ) +
int( deltaOfpGraph3 ) +
int( deltaOfpGraph4 ) +
int( deltaOfpGraph5 ) +
int( deltaOfpGraph6 ) +
int( deltaOfpGraph7 ) ) / clusterCount
if avgDeltaOfpGraph > thresholdMin \
and avgDeltaOfpGraph < thresholdMax\
and int( i ) > iterIgnore:
latencyOfpToGraphList.append( avgDeltaOfpGraph )
elif avgDeltaOfpGraph > ( -10 ) and \
avgDeltaOfpGraph < 0.0 and\
int( i ) > iterIgnore:
main.log.info( "Sub-millisecond result likely; " +
"negative result was rounded to 0" )
# NOTE: Current metrics framework does not
# support sub-millisecond accuracy. Therefore,
# if the result is negative, we can reasonably
# conclude sub-millisecond results and just
# append the best rounded effort - 0 ms.
latencyOfpToGraphList.append( 0 )
else:
main.log.info( "Results for ofp-to-graph " +
"ignored due to excess in threshold" )
avgDeltaOfpDevice = \
( float( deltaOfpDevice1 ) +
float( deltaOfpDevice2 ) +
float( deltaOfpDevice3 ) +
float( deltaOfpDevice4 ) +
float( deltaOfpDevice5 ) +
float( deltaOfpDevice6 ) +
float( deltaOfpDevice7 ) ) / clusterCount
# NOTE: ofp - delta measurements are occasionally negative
# due to system time misalignment.
latencyOfpToDeviceList.append( avgDeltaOfpDevice )
deltaOfpTcp = int( t0Ofp ) - int( t0Tcp )
if deltaOfpTcp > thresholdMin \
and deltaOfpTcp < thresholdMax and\
int( i ) > iterIgnore:
latencyTcpToOfpList.append( deltaOfpTcp )
else:
main.log.info( "Results fo tcp-to-ofp " +
"ignored due to excess in threshold" )
# TODO:
# Fetch logs upon threshold excess
main.log.info( "ONOS1 delta end-to-end: " +
str( deltaGraph1 ) + " ms" )
main.log.info( "ONOS1 delta OFP - graph: " +
str( deltaOfpGraph1 ) + " ms" )
main.log.info( "ONOS1 delta device - t0: " +
str( deltaDevice1 ) + " ms" )
main.log.info( "TCP to OFP delta: " +
str( deltaOfpTcp ) + " ms" )
main.step( "Remove switch from controller" )
main.Mininet1.deleteSwController( "s1" )
time.sleep( 5 )
# END of for loop iteration
# If there is at least 1 element in each list,
# pass the test case
if len( latencyEndToEndList ) > 0 and\
len( latencyOfpToGraphList ) > 0 and\
len( latencyOfpToDeviceList ) > 0 and\
len( latencyT0ToDeviceList ) > 0 and\
len( latencyTcpToOfpList ) > 0:
assertion = main.TRUE
elif len( latencyEndToEndList ) == 0:
# The appending of 0 here is to prevent
# the min,max,sum functions from failing
# below
latencyEndToEndList.append( 0 )
assertion = main.FALSE
elif len( latencyOfpToGraphList ) == 0:
latencyOfpToGraphList.append( 0 )
assertion = main.FALSE
elif len( latencyOfpToDeviceList ) == 0:
latencyOfpToDeviceList.append( 0 )
assertion = main.FALSE
elif len( latencyT0ToDeviceList ) == 0:
latencyT0ToDeviceList.append( 0 )
assertion = main.FALSE
elif len( latencyTcpToOfpList ) == 0:
latencyTcpToOfpList.append( 0 )
assertion = main.FALSE
# Calculate min, max, avg of latency lists
latencyEndToEndMax = \
int( max( latencyEndToEndList ) )
latencyEndToEndMin = \
int( min( latencyEndToEndList ) )
latencyEndToEndAvg = \
( int( sum( latencyEndToEndList ) ) /
len( latencyEndToEndList ) )
latencyEndToEndStdDev = \
str( round( numpy.std( latencyEndToEndList ), 1 ) )
latencyOfpToGraphMax = \
int( max( latencyOfpToGraphList ) )
latencyOfpToGraphMin = \
int( min( latencyOfpToGraphList ) )
latencyOfpToGraphAvg = \
( int( sum( latencyOfpToGraphList ) ) /
len( latencyOfpToGraphList ) )
latencyOfpToGraphStdDev = \
str( round( numpy.std( latencyOfpToGraphList ), 1 ) )
latencyOfpToDeviceMax = \
int( max( latencyOfpToDeviceList ) )
latencyOfpToDeviceMin = \
int( min( latencyOfpToDeviceList ) )
latencyOfpToDeviceAvg = \
( int( sum( latencyOfpToDeviceList ) ) /
len( latencyOfpToDeviceList ) )
latencyOfpToDeviceStdDev = \
str( round( numpy.std( latencyOfpToDeviceList ), 1 ) )
latencyT0ToDeviceMax = \
int( max( latencyT0ToDeviceList ) )
latencyT0ToDeviceMin = \
int( min( latencyT0ToDeviceList ) )
latencyT0ToDeviceAvg = \
( int( sum( latencyT0ToDeviceList ) ) /
len( latencyT0ToDeviceList ) )
latencyOfpToDeviceStdDev = \
str( round( numpy.std( latencyT0ToDeviceList ), 1 ) )
latencyTcpToOfpMax = \
int( max( latencyTcpToOfpList ) )
latencyTcpToOfpMin = \
int( min( latencyTcpToOfpList ) )
latencyTcpToOfpAvg = \
( int( sum( latencyTcpToOfpList ) ) /
len( latencyTcpToOfpList ) )
latencyTcpToOfpStdDev = \
str( round( numpy.std( latencyTcpToOfpList ), 1 ) )
main.log.report( "Cluster size: " + str( clusterCount ) +
" node(s)" )
main.log.report( "Switch add - End-to-end latency: " +
"Avg: " + str( latencyEndToEndAvg ) + " ms " +
"Std Deviation: " + latencyEndToEndStdDev + " ms" )
main.log.report(
"Switch add - OFP-to-Graph latency: " +
"Note: results are not accurate to sub-millisecond. " +
"Any sub-millisecond results are rounded to 0 ms. " )
main.log.report( "Avg: " + str( latencyOfpToGraphAvg ) + " ms " +
"Std Deviation: " + latencyOfpToGraphStdDev + " ms" )
main.log.report( "Switch add - TCP-to-OFP latency: " +
"Avg: " + str( latencyTcpToOfpAvg ) + " ms " +
"Std Deviation: " + latencyTcpToOfpStdDev + " ms" )
if debugMode == 'on':
main.ONOS1.cpLogsToDir( "/opt/onos/log/karaf.log",
"/tmp/", copyFileName="sw_lat_karaf" )
utilities.assert_equals( expect=main.TRUE, actual=assertion,
onpass="Switch latency test successful",
onfail="Switch latency test failed" )
def CASE3( self, main ):
"""
Bring port up / down and measure latency.
Port enable / disable is simulated by ifconfig up / down
In ONOS-next, we must ensure that the port we are
manipulating is connected to another switch with a valid
connection. Otherwise, graph view will not be updated.
"""
import time
import subprocess
import os
import requests
import json
import numpy
global clusterCount
ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
ONOSUser = main.params[ 'CTRL' ][ 'user' ]
defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
assertion = main.TRUE
# Number of iterations of case
numIter = main.params[ 'TEST' ][ 'numIter' ]
# Timestamp 'keys' for json metrics output.
# These are subject to change, hence moved into params
deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
debugMode = main.params[ 'TEST' ][ 'debugMode' ]
localTime = time.strftime( '%x %X' )
localTime = localTime.replace( "/", "" )
localTime = localTime.replace( " ", "_" )
localTime = localTime.replace( ":", "" )
if debugMode == 'on':
main.ONOS1.tsharkPcap( "eth0",
"/tmp/port_lat_pcap_" + localTime )
# Threshold for this test case
upThresholdStr = main.params[ 'TEST' ][ 'portUpThreshold' ]
downThresholdStr = main.params[ 'TEST' ][ 'portDownThreshold' ]
upThresholdObj = upThresholdStr.split( "," )
downThresholdObj = downThresholdStr.split( "," )
upThresholdMin = int( upThresholdObj[ 0 ] )
upThresholdMax = int( upThresholdObj[ 1 ] )
downThresholdMin = int( downThresholdObj[ 0 ] )
downThresholdMax = int( downThresholdObj[ 1 ] )
# NOTE: Some hardcoded variables you may need to configure
# besides the params
tsharkPortStatus = "OFP 130 Port Status"
tsharkPortUp = "/tmp/tshark_port_up.txt"
tsharkPortDown = "/tmp/tshark_port_down.txt"
interfaceConfig = "s1-eth1"
main.log.report( "Port enable / disable latency" )
main.log.report( "Simulated by ifconfig up / down" )
main.log.report( "Total iterations of test: " + str( numIter ) )
main.step( "Assign switches s1 and s2 to controller 1" )
main.Mininet1.assignSwController( sw="1", ip1=ONOS1Ip,
port1=defaultSwPort )
main.Mininet1.assignSwController( sw="2", ip1=ONOS1Ip,
port1=defaultSwPort )
# Give enough time for metrics to propagate the
# assign controller event. Otherwise, these events may
# carry over to our measurements
time.sleep( 15 )
portUpDeviceToOfpList = []
portUpGraphToOfpList = []
portDownDeviceToOfpList = []
portDownGraphToOfpList = []
for i in range( 0, int( numIter ) ):
main.step( "Starting wireshark capture for port status down" )
main.ONOS1.tsharkGrep( tsharkPortStatus,
tsharkPortDown )
time.sleep( 5 )
# Disable interface that is connected to switch 2
main.step( "Disable port: " + interfaceConfig )
main.Mininet1.handle.sendline( "sh ifconfig " +
interfaceConfig + " down" )
main.Mininet1.handle.expect( "mininet>" )
time.sleep( 3 )
main.ONOS1.tsharkStop()
# Copy tshark output file from ONOS to TestON instance
#/tmp directory
os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
tsharkPortDown + " /tmp/" )
fPortDown = open( tsharkPortDown, 'r' )
# Get first line of port down event from tshark
fLine = fPortDown.readline()
objDown = fLine.split( " " )
if len( fLine ) > 0:
# NOTE: objDown[ 1 ] is a very unreliable
# way to determine the timestamp. If
# results seem way off, check the object
# itself by printing it out
timestampBeginPtDown = int( float( objDown[ 1 ] ) * 1000 )
# For some reason, wireshark decides to record the
# timestamp at the 3rd object position instead of
# 2nd at unpredictable times. This statement is
# used to capture that odd behavior and use the
# correct epoch time
if timestampBeginPtDown < 1400000000000:
timestampBeginPtDown = \
int( float( objDown[ 2 ] ) * 1000 )
main.log.info( "Port down begin timestamp: " +
str( timestampBeginPtDown ) )
else:
main.log.info( "Tshark output file returned unexpected" +
" results: " + str( objDown ) )
timestampBeginPtDown = 0
fPortDown.close()
main.step( "Obtain t1 by metrics call" )
jsonStrUp1 = main.ONOS1cli.topologyEventsMetrics()
jsonObj1 = json.loads( jsonStrUp1 )
# Obtain graph timestamp. This timestsamp captures
# the epoch time at which the topology graph was updated.
graphTimestamp1 = \
jsonObj1[ graphTimestamp ][ 'value' ]
# Obtain device timestamp. This timestamp captures
# the epoch time at which the device event happened
deviceTimestamp1 = \
jsonObj1[ deviceTimestamp ][ 'value' ]
# Get delta between graph event and OFP
ptDownGraphToOfp1 = int( graphTimestamp1 ) -\
int( timestampBeginPtDown )
# Get delta between device event and OFP
ptDownDeviceToOfp1 = int( deviceTimestamp1 ) -\
int( timestampBeginPtDown )
if clusterCount >= 3:
jsonStrUp2 = main.ONOS2cli.topologyEventsMetrics()
jsonStrUp3 = main.ONOS3cli.topologyEventsMetrics()
jsonObj2 = json.loads( jsonStrUp2 )
jsonObj3 = json.loads( jsonStrUp3 )
graphTimestamp2 = \
jsonObj2[ graphTimestamp ][ 'value' ]
graphTimestamp3 = \
jsonObj3[ graphTimestamp ][ 'value' ]
deviceTimestamp2 = \
jsonObj2[ deviceTimestamp ][ 'value' ]
deviceTimestamp3 = \
jsonObj3[ deviceTimestamp ][ 'value' ]
ptDownGraphToOfp2 = int( graphTimestamp2 ) -\
int( timestampBeginPtDown )
ptDownGraphToOfp3 = int( graphTimestamp3 ) -\
int( timestampBeginPtDown )
ptDownDeviceToOfp2 = int( deviceTimestamp2 ) -\
int( timestampBeginPtDown )
ptDownDeviceToOfp3 = int( deviceTimestamp3 ) -\
int( timestampBeginPtDown )
else:
ptDownGraphToOfp2 = 0
ptDownGraphToOfp3 = 0
ptDownDeviceToOfp2 = 0
ptDownDeviceToOfp3 = 0
if clusterCount >= 5:
jsonStrUp4 = main.ONOS4cli.topologyEventsMetrics()
jsonStrUp5 = main.ONOS5cli.topologyEventsMetrics()
jsonObj4 = json.loads( jsonStrUp4 )
jsonObj5 = json.loads( jsonStrUp5 )
graphTimestamp4 = \
jsonObj4[ graphTimestamp ][ 'value' ]
graphTimestamp5 = \
jsonObj5[ graphTimestamp ][ 'value' ]
deviceTimestamp4 = \
jsonObj4[ deviceTimestamp ][ 'value' ]
deviceTimestamp5 = \
jsonObj5[ deviceTimestamp ][ 'value' ]
ptDownGraphToOfp4 = int( graphTimestamp4 ) -\
int( timestampBeginPtDown )
ptDownGraphToOfp5 = int( graphTimestamp5 ) -\
int( timestampBeginPtDown )
ptDownDeviceToOfp4 = int( deviceTimestamp4 ) -\
int( timestampBeginPtDown )
ptDownDeviceToOfp5 = int( deviceTimestamp5 ) -\
int( timestampBeginPtDown )
else:
ptDownGraphToOfp4 = 0
ptDownGraphToOfp5 = 0
ptDownDeviceToOfp4 = 0
ptDownDeviceToOfp5 = 0
if clusterCount >= 7:
jsonStrUp6 = main.ONOS6cli.topologyEventsMetrics()
jsonStrUp7 = main.ONOS7cli.topologyEventsMetrics()
jsonObj6 = json.loads( jsonStrUp6 )
jsonObj7 = json.loads( jsonStrUp7 )
graphTimestamp6 = \
jsonObj6[ graphTimestamp ][ 'value' ]
graphTimestamp7 = \
jsonObj7[ graphTimestamp ][ 'value' ]
deviceTimestamp6 = \
jsonObj6[ deviceTimestamp ][ 'value' ]
deviceTimestamp7 = \
jsonObj7[ deviceTimestamp ][ 'value' ]
ptDownGraphToOfp6 = int( graphTimestamp6 ) -\
int( timestampBeginPtDown )
ptDownGraphToOfp7 = int( graphTimestamp7 ) -\
int( timestampBeginPtDown )
ptDownDeviceToOfp6 = int( deviceTimestamp6 ) -\
int( timestampBeginPtDown )
ptDownDeviceToOfp7 = int( deviceTimestamp7 ) -\
int( timestampBeginPtDown )
else:
ptDownGraphToOfp6 = 0
ptDownGraphToOfp7 = 0
ptDownDeviceToOfp6 = 0
ptDownDeviceToOfp7 = 0
time.sleep( 3 )
# Caluclate average across clusters
ptDownGraphToOfpAvg =\
( int( ptDownGraphToOfp1 ) +
int( ptDownGraphToOfp2 ) +
int( ptDownGraphToOfp3 ) +
int( ptDownGraphToOfp4 ) +
int( ptDownGraphToOfp5 ) +
int( ptDownGraphToOfp6 ) +
int( ptDownGraphToOfp7 ) ) / clusterCount
ptDownDeviceToOfpAvg = \
( int( ptDownDeviceToOfp1 ) +
int( ptDownDeviceToOfp2 ) +
int( ptDownDeviceToOfp3 ) +
int( ptDownDeviceToOfp4 ) +
int( ptDownDeviceToOfp5 ) +
int( ptDownDeviceToOfp6 ) +
int( ptDownDeviceToOfp7 ) ) / clusterCount
if ptDownGraphToOfpAvg > downThresholdMin and \
ptDownGraphToOfpAvg < downThresholdMax:
portDownGraphToOfpList.append(
ptDownGraphToOfpAvg )
main.log.info( "Port down: graph to ofp avg: " +
str( ptDownGraphToOfpAvg ) + " ms" )
else:
main.log.info( "Average port down graph-to-ofp result" +
" exceeded the threshold: " +
str( ptDownGraphToOfpAvg ) )
if ptDownDeviceToOfpAvg > 0 and \
ptDownDeviceToOfpAvg < 1000:
portDownDeviceToOfpList.append(
ptDownDeviceToOfpAvg )
main.log.info( "Port down: device to ofp avg: " +
str( ptDownDeviceToOfpAvg ) + " ms" )
else:
main.log.info( "Average port down device-to-ofp result" +
" exceeded the threshold: " +
str( ptDownDeviceToOfpAvg ) )
# Port up events
main.step( "Enable port and obtain timestamp" )
main.step( "Starting wireshark capture for port status up" )
main.ONOS1.tsharkGrep( tsharkPortStatus, tsharkPortUp )
time.sleep( 5 )
main.Mininet1.handle.sendline( "sh ifconfig " +
interfaceConfig + " up" )
main.Mininet1.handle.expect( "mininet>" )
# Allow time for tshark to capture event
time.sleep( 5 )
main.ONOS1.tsharkStop()
time.sleep( 3 )
os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
tsharkPortUp + " /tmp/" )
fPortUp = open( tsharkPortUp, 'r' )
fLine = fPortUp.readline()
objUp = fLine.split( " " )
if len( fLine ) > 0:
timestampBeginPtUp = int( float( objUp[ 1 ] ) * 1000 )
if timestampBeginPtUp < 1400000000000:
timestampBeginPtUp = \
int( float( objUp[ 2 ] ) * 1000 )
main.log.info( "Port up begin timestamp: " +
str( timestampBeginPtUp ) )
else:
main.log.info( "Tshark output file returned unexpected" +
" results." )
timestampBeginPtUp = 0
fPortUp.close()
# Obtain metrics shortly afterwards
# This timestsamp captures
# the epoch time at which the topology graph was updated.
main.step( "Obtain t1 by REST call" )
jsonStrUp1 = main.ONOS1cli.topologyEventsMetrics()
jsonObj1 = json.loads( jsonStrUp1 )
graphTimestamp1 = \
jsonObj1[ graphTimestamp ][ 'value' ]
# Obtain device timestamp. This timestamp captures
# the epoch time at which the device event happened
deviceTimestamp1 = \
jsonObj1[ deviceTimestamp ][ 'value' ]
# Get delta between graph event and OFP
ptUpGraphToOfp1 = int( graphTimestamp1 ) -\
int( timestampBeginPtUp )
# Get delta between device event and OFP
ptUpDeviceToOfp1 = int( deviceTimestamp1 ) -\
int( timestampBeginPtUp )
if clusterCount >= 3:
jsonStrUp2 = main.ONOS2cli.topologyEventsMetrics()
jsonStrUp3 = main.ONOS3cli.topologyEventsMetrics()
jsonObj2 = json.loads( jsonStrUp2 )
jsonObj3 = json.loads( jsonStrUp3 )
graphTimestamp2 = \
jsonObj2[ graphTimestamp ][ 'value' ]
graphTimestamp3 = \
jsonObj3[ graphTimestamp ][ 'value' ]
deviceTimestamp2 = \
jsonObj2[ deviceTimestamp ][ 'value' ]
deviceTimestamp3 = \
jsonObj3[ deviceTimestamp ][ 'value' ]
ptUpGraphToOfp2 = int( graphTimestamp2 ) -\
int( timestampBeginPtUp )
ptUpGraphToOfp3 = int( graphTimestamp3 ) -\
int( timestampBeginPtUp )
ptUpDeviceToOfp2 = int( deviceTimestamp2 ) -\
int( timestampBeginPtUp )
ptUpDeviceToOfp3 = int( deviceTimestamp3 ) -\
int( timestampBeginPtUp )
else:
ptUpGraphToOfp2 = 0
ptUpGraphToOfp3 = 0
ptUpDeviceToOfp2 = 0
ptUpDeviceToOfp3 = 0
if clusterCount >= 5:
jsonStrUp4 = main.ONOS4cli.topologyEventsMetrics()
jsonStrUp5 = main.ONOS5cli.topologyEventsMetrics()
jsonObj4 = json.loads( jsonStrUp4 )
jsonObj5 = json.loads( jsonStrUp5 )
graphTimestamp4 = \
jsonObj4[ graphTimestamp ][ 'value' ]
graphTimestamp5 = \
jsonObj5[ graphTimestamp ][ 'value' ]
deviceTimestamp4 = \
jsonObj4[ deviceTimestamp ][ 'value' ]
deviceTimestamp5 = \
jsonObj5[ deviceTimestamp ][ 'value' ]
ptUpGraphToOfp4 = int( graphTimestamp4 ) -\
int( timestampBeginPtUp )
ptUpGraphToOfp5 = int( graphTimestamp5 ) -\
int( timestampBeginPtUp )
ptUpDeviceToOfp4 = int( deviceTimestamp4 ) -\
int( timestampBeginPtUp )
ptUpDeviceToOfp5 = int( deviceTimestamp5 ) -\
int( timestampBeginPtUp )
else:
ptUpGraphToOfp4 = 0
ptUpGraphToOfp5 = 0
ptUpDeviceToOfp4 = 0
ptUpDeviceToOfp5 = 0
if clusterCount >= 7:
jsonStrUp6 = main.ONOS6cli.topologyEventsMetrics()
jsonStrUp7 = main.ONOS7cli.topologyEventsMetrics()
jsonObj6 = json.loads( jsonStrUp6 )
jsonObj7 = json.loads( jsonStrUp7 )
graphTimestamp6 = \
jsonObj6[ graphTimestamp ][ 'value' ]
graphTimestamp7 = \
jsonObj7[ graphTimestamp ][ 'value' ]
deviceTimestamp6 = \
jsonObj6[ deviceTimestamp ][ 'value' ]
deviceTimestamp7 = \
jsonObj7[ deviceTimestamp ][ 'value' ]
ptUpGraphToOfp6 = int( graphTimestamp6 ) -\
int( timestampBeginPtUp )
ptUpGraphToOfp7 = int( graphTimestamp7 ) -\
int( timestampBeginPtUp )
ptUpDeviceToOfp6 = int( deviceTimestamp6 ) -\
int( timestampBeginPtUp )
ptUpDeviceToOfp7 = int( deviceTimestamp7 ) -\
int( timestampBeginPtUp )
else:
ptUpGraphToOfp6 = 0
ptUpGraphToOfp7 = 0
ptUpDeviceToOfp6 = 0
ptUpDeviceToOfp7 = 0
ptUpGraphToOfpAvg = \
( int( ptUpGraphToOfp1 ) +
int( ptUpGraphToOfp2 ) +
int( ptUpGraphToOfp3 ) +
int( ptUpGraphToOfp4 ) +
int( ptUpGraphToOfp5 ) +
int( ptUpGraphToOfp6 ) +
int( ptUpGraphToOfp7 ) ) / clusterCount
ptUpDeviceToOfpAvg = \
( int( ptUpDeviceToOfp1 ) +
int( ptUpDeviceToOfp2 ) +
int( ptUpDeviceToOfp3 ) +
int( ptUpDeviceToOfp4 ) +
int( ptUpDeviceToOfp5 ) +
int( ptUpDeviceToOfp6 ) +
int( ptUpDeviceToOfp7 ) ) / clusterCount
if ptUpGraphToOfpAvg > upThresholdMin and \
ptUpGraphToOfpAvg < upThresholdMax:
portUpGraphToOfpList.append(
ptUpGraphToOfpAvg )
main.log.info( "Port down: graph to ofp avg: " +
str( ptUpGraphToOfpAvg ) + " ms" )
else:
main.log.info( "Average port up graph-to-ofp result" +
" exceeded the threshold: " +
str( ptUpGraphToOfpAvg ) )
if ptUpDeviceToOfpAvg > upThresholdMin and \
ptUpDeviceToOfpAvg < upThresholdMax:
portUpDeviceToOfpList.append(
ptUpDeviceToOfpAvg )
main.log.info( "Port up: device to ofp avg: " +
str( ptUpDeviceToOfpAvg ) + " ms" )
else:
main.log.info( "Average port up device-to-ofp result" +
" exceeded the threshold: " +
str( ptUpDeviceToOfpAvg ) )
# END ITERATION FOR LOOP
# Check all list for latency existence and set assertion
if ( portDownGraphToOfpList and portDownDeviceToOfpList
and portUpGraphToOfpList and portUpDeviceToOfpList ):
assertion = main.TRUE
main.log.report( "Cluster size: " + str( clusterCount ) +
" node(s)" )
# Calculate and report latency measurements
portDownGraphToOfpMin = min( portDownGraphToOfpList )
portDownGraphToOfpMax = max( portDownGraphToOfpList )
portDownGraphToOfpAvg = \
( sum( portDownGraphToOfpList ) /
len( portDownGraphToOfpList ) )
portDownGraphToOfpStdDev = \
str( round( numpy.std( portDownGraphToOfpList ), 1 ) )
main.log.report( "Port down graph-to-ofp " +
"Avg: " + str( portDownGraphToOfpAvg ) + " ms " +
"Std Deviation: " + portDownGraphToOfpStdDev + " ms" )
portDownDeviceToOfpMin = min( portDownDeviceToOfpList )
portDownDeviceToOfpMax = max( portDownDeviceToOfpList )
portDownDeviceToOfpAvg = \
( sum( portDownDeviceToOfpList ) /
len( portDownDeviceToOfpList ) )
portDownDeviceToOfpStdDev = \
str( round( numpy.std( portDownDeviceToOfpList ), 1 ) )
main.log.report(
"Port down device-to-ofp " +
"Avg: " +
str( portDownDeviceToOfpAvg ) +
" ms " +
"Std Deviation: " +
portDownDeviceToOfpStdDev +
" ms" )
portUpGraphToOfpMin = min( portUpGraphToOfpList )
portUpGraphToOfpMax = max( portUpGraphToOfpList )
portUpGraphToOfpAvg = \
( sum( portUpGraphToOfpList ) /
len( portUpGraphToOfpList ) )
portUpGraphToOfpStdDev = \
str( round( numpy.std( portUpGraphToOfpList ), 1 ) )
main.log.report( "Port up graph-to-ofp " +
"Avg: " + str( portUpGraphToOfpAvg ) + " ms " +
"Std Deviation: " + portUpGraphToOfpStdDev + " ms" )
portUpDeviceToOfpMin = min( portUpDeviceToOfpList )
portUpDeviceToOfpMax = max( portUpDeviceToOfpList )
portUpDeviceToOfpAvg = \
( sum( portUpDeviceToOfpList ) /
len( portUpDeviceToOfpList ) )
portUpDeviceToOfpStdDev = \
str( round( numpy.std( portUpDeviceToOfpList ), 1 ) )
main.log.report( "Port up device-to-ofp " +
"Avg: " + str( portUpDeviceToOfpAvg ) + " ms " +
"Std Deviation: " + portUpDeviceToOfpStdDev + " ms" )
# Remove switches from controller for next test
main.Mininet1.deleteSwController( "s1" )
main.Mininet1.deleteSwController( "s2" )
utilities.assert_equals(
expect=main.TRUE,
actual=assertion,
onpass="Port discovery latency calculation successful",
onfail="Port discovery test failed" )
def CASE4( self, main ):
"""
Link down event using loss rate 100%
Important:
Use a simple 2 switch topology with 1 link between
the two switches. Ensure that mac addresses of the
switches are 1 / 2 respectively
"""
import time
import subprocess
import os
import requests
import json
import numpy
ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
ONOSUser = main.params[ 'CTRL' ][ 'user' ]
defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
# Number of iterations of case
numIter = main.params[ 'TEST' ][ 'numIter' ]
# Timestamp 'keys' for json metrics output.
# These are subject to change, hence moved into params
deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
linkTimestamp = main.params[ 'JSON' ][ 'linkTimestamp' ]
graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
debugMode = main.params[ 'TEST' ][ 'debugMode' ]
localTime = time.strftime( '%x %X' )
localTime = localTime.replace( "/", "" )
localTime = localTime.replace( " ", "_" )
localTime = localTime.replace( ":", "" )
if debugMode == 'on':
main.ONOS1.tsharkPcap( "eth0",
"/tmp/link_lat_pcap_" + localTime )
# Threshold for this test case
upThresholdStr = main.params[ 'TEST' ][ 'linkUpThreshold' ]
downThresholdStr = main.params[ 'TEST' ][ 'linkDownThreshold' ]
upThresholdObj = upThresholdStr.split( "," )
downThresholdObj = downThresholdStr.split( "," )
upThresholdMin = int( upThresholdObj[ 0 ] )
upThresholdMax = int( upThresholdObj[ 1 ] )
downThresholdMin = int( downThresholdObj[ 0 ] )
downThresholdMax = int( downThresholdObj[ 1 ] )
assertion = main.TRUE
# Link event timestamp to system time list
linkDownLinkToSystemList = []
linkUpLinkToSystemList = []
# Graph event timestamp to system time list
linkDownGraphToSystemList = []
linkUpGraphToSystemList = []
main.log.report( "Link up / down discovery latency between " +
"two switches" )
main.log.report( "Simulated by setting loss-rate 100%" )
main.log.report( "'tc qdisc add dev <intfs> root netem loss 100%'" )
main.log.report( "Total iterations of test: " + str( numIter ) )
main.step( "Assign all switches" )
main.Mininet1.assignSwController( sw="1",
ip1=ONOS1Ip, port1=defaultSwPort )
main.Mininet1.assignSwController( sw="2",
ip1=ONOS1Ip, port1=defaultSwPort )
main.step( "Verifying switch assignment" )
resultS1 = main.Mininet1.getSwController( sw="s1" )
resultS2 = main.Mininet1.getSwController( sw="s2" )
# Allow time for events to finish before taking measurements
time.sleep( 10 )
linkDown1 = False
linkDown2 = False
linkDown3 = False
# Start iteration of link event test
for i in range( 0, int( numIter ) ):
main.step( "Getting initial system time as t0" )
# System time in epoch ms
timestampLinkDownT0 = time.time() * 1000
# Link down is simulated by 100% loss rate using traffic
# control command
main.Mininet1.handle.sendline(
"sh tc qdisc add dev s1-eth1 root netem loss 100%" )
# TODO: Iterate through 'links' command to verify that
# link s1 -> s2 went down ( loop timeout 30 seconds )
# on all 3 ONOS instances
main.log.info( "Checking ONOS for link update" )
loopCount = 0
while( not ( linkDown1 and linkDown2 and linkDown3 )
and loopCount < 30 ):
jsonStr1 = main.ONOS1cli.links()
jsonStr2 = main.ONOS2cli.links()
jsonStr3 = main.ONOS3cli.links()
if not ( jsonStr1 and jsonStr2 and jsonStr3 ):
main.log.error( "CLI command returned error " )
break
else:
jsonObj1 = json.loads( jsonStr1 )
jsonObj2 = json.loads( jsonStr2 )
jsonObj3 = json.loads( jsonStr3 )
for obj1 in jsonObj1:
if '01' not in obj1[ 'src' ][ 'device' ]:
linkDown1 = True
main.log.info( "Link down from " +
"s1 -> s2 on ONOS1 detected" )
for obj2 in jsonObj2:
if '01' not in obj2[ 'src' ][ 'device' ]:
linkDown2 = True
main.log.info( "Link down from " +
"s1 -> s2 on ONOS2 detected" )
for obj3 in jsonObj3:
if '01' not in obj3[ 'src' ][ 'device' ]:
linkDown3 = True
main.log.info( "Link down from " +
"s1 -> s2 on ONOS3 detected" )
loopCount += 1
# If CLI doesn't like the continuous requests
# and exits in this loop, increase the sleep here.
# Consequently, while loop timeout will increase
time.sleep( 1 )
# Give time for metrics measurement to catch up
# NOTE: May need to be configured more accurately
time.sleep( 10 )
# If we exited the while loop and link down 1,2,3 are still
# false, then ONOS has failed to discover link down event
if not ( linkDown1 and linkDown2 and linkDown3 ):
main.log.info( "Link down discovery failed" )
linkDownLatGraph1 = 0
linkDownLatGraph2 = 0
linkDownLatGraph3 = 0
linkDownLatDevice1 = 0
linkDownLatDevice2 = 0
linkDownLatDevice3 = 0
assertion = main.FALSE
else:
jsonTopoMetrics1 =\
main.ONOS1cli.topologyEventsMetrics()
jsonTopoMetrics2 =\
main.ONOS2cli.topologyEventsMetrics()
jsonTopoMetrics3 =\
main.ONOS3cli.topologyEventsMetrics()
jsonTopoMetrics1 = json.loads( jsonTopoMetrics1 )
jsonTopoMetrics2 = json.loads( jsonTopoMetrics2 )
jsonTopoMetrics3 = json.loads( jsonTopoMetrics3 )
main.log.info( "Obtaining graph and device timestamp" )
graphTimestamp1 = \
jsonTopoMetrics1[ graphTimestamp ][ 'value' ]
graphTimestamp2 = \
jsonTopoMetrics2[ graphTimestamp ][ 'value' ]
graphTimestamp3 = \
jsonTopoMetrics3[ graphTimestamp ][ 'value' ]
linkTimestamp1 = \
jsonTopoMetrics1[ linkTimestamp ][ 'value' ]
linkTimestamp2 = \
jsonTopoMetrics2[ linkTimestamp ][ 'value' ]
linkTimestamp3 = \
jsonTopoMetrics3[ linkTimestamp ][ 'value' ]
if graphTimestamp1 and graphTimestamp2 and\
graphTimestamp3 and linkTimestamp1 and\
linkTimestamp2 and linkTimestamp3:
linkDownLatGraph1 = int( graphTimestamp1 ) -\
int( timestampLinkDownT0 )
linkDownLatGraph2 = int( graphTimestamp2 ) -\
int( timestampLinkDownT0 )
linkDownLatGraph3 = int( graphTimestamp3 ) -\
int( timestampLinkDownT0 )
linkDownLatLink1 = int( linkTimestamp1 ) -\
int( timestampLinkDownT0 )
linkDownLatLink2 = int( linkTimestamp2 ) -\
int( timestampLinkDownT0 )
linkDownLatLink3 = int( linkTimestamp3 ) -\
int( timestampLinkDownT0 )
else:
main.log.error( "There was an error calculating" +
" the delta for link down event" )
linkDownLatGraph1 = 0
linkDownLatGraph2 = 0
linkDownLatGraph3 = 0
linkDownLatDevice1 = 0
linkDownLatDevice2 = 0
linkDownLatDevice3 = 0
main.log.info( "Link down latency ONOS1 iteration " +
str( i ) + " (end-to-end): " +
str( linkDownLatGraph1 ) + " ms" )
main.log.info( "Link down latency ONOS2 iteration " +
str( i ) + " (end-to-end): " +
str( linkDownLatGraph2 ) + " ms" )
main.log.info( "Link down latency ONOS3 iteration " +
str( i ) + " (end-to-end): " +
str( linkDownLatGraph3 ) + " ms" )
main.log.info( "Link down latency ONOS1 iteration " +
str( i ) + " (link-event-to-system-timestamp): " +
str( linkDownLatLink1 ) + " ms" )
main.log.info( "Link down latency ONOS2 iteration " +
str( i ) + " (link-event-to-system-timestamp): " +
str( linkDownLatLink2 ) + " ms" )
main.log.info( "Link down latency ONOS3 iteration " +
str( i ) + " (link-event-to-system-timestamp): " +
str( linkDownLatLink3 ) )
# Calculate avg of node calculations
linkDownLatGraphAvg =\
( linkDownLatGraph1 +
linkDownLatGraph2 +
linkDownLatGraph3 ) / 3
linkDownLatLinkAvg =\
( linkDownLatLink1 +
linkDownLatLink2 +
linkDownLatLink3 ) / 3
# Set threshold and append latency to list
if linkDownLatGraphAvg > downThresholdMin and\
linkDownLatGraphAvg < downThresholdMax:
linkDownGraphToSystemList.append(
linkDownLatGraphAvg )
else:
main.log.info( "Link down latency exceeded threshold" )
main.log.info( "Results for iteration " + str( i ) +
"have been omitted" )
if linkDownLatLinkAvg > downThresholdMin and\
linkDownLatLinkAvg < downThresholdMax:
linkDownLinkToSystemList.append(
linkDownLatLinkAvg )
else:
main.log.info( "Link down latency exceeded threshold" )
main.log.info( "Results for iteration " + str( i ) +
"have been omitted" )
# NOTE: To remove loss rate and measure latency:
# 'sh tc qdisc del dev s1-eth1 root'
timestampLinkUpT0 = time.time() * 1000
main.Mininet1.handle.sendline( "sh tc qdisc del dev " +
"s1-eth1 root" )
main.Mininet1.handle.expect( "mininet>" )
main.log.info( "Checking ONOS for link update" )
linkDown1 = True
linkDown2 = True
linkDown3 = True
loopCount = 0
while( ( linkDown1 and linkDown2 and linkDown3 )
and loopCount < 30 ):
jsonStr1 = main.ONOS1cli.links()
jsonStr2 = main.ONOS2cli.links()
jsonStr3 = main.ONOS3cli.links()
if not ( jsonStr1 and jsonStr2 and jsonStr3 ):
main.log.error( "CLI command returned error " )
break
else:
jsonObj1 = json.loads( jsonStr1 )
jsonObj2 = json.loads( jsonStr2 )
jsonObj3 = json.loads( jsonStr3 )
for obj1 in jsonObj1:
if '01' in obj1[ 'src' ][ 'device' ]:
linkDown1 = False
main.log.info( "Link up from " +
"s1 -> s2 on ONOS1 detected" )
for obj2 in jsonObj2:
if '01' in obj2[ 'src' ][ 'device' ]:
linkDown2 = False
main.log.info( "Link up from " +
"s1 -> s2 on ONOS2 detected" )
for obj3 in jsonObj3:
if '01' in obj3[ 'src' ][ 'device' ]:
linkDown3 = False
main.log.info( "Link up from " +
"s1 -> s2 on ONOS3 detected" )
loopCount += 1
time.sleep( 1 )
if ( linkDown1 and linkDown2 and linkDown3 ):
main.log.info( "Link up discovery failed" )
linkUpLatGraph1 = 0
linkUpLatGraph2 = 0
linkUpLatGraph3 = 0
linkUpLatDevice1 = 0
linkUpLatDevice2 = 0
linkUpLatDevice3 = 0
assertion = main.FALSE
else:
jsonTopoMetrics1 =\
main.ONOS1cli.topologyEventsMetrics()
jsonTopoMetrics2 =\
main.ONOS2cli.topologyEventsMetrics()
jsonTopoMetrics3 =\
main.ONOS3cli.topologyEventsMetrics()
jsonTopoMetrics1 = json.loads( jsonTopoMetrics1 )
jsonTopoMetrics2 = json.loads( jsonTopoMetrics2 )
jsonTopoMetrics3 = json.loads( jsonTopoMetrics3 )
main.log.info( "Obtaining graph and device timestamp" )
graphTimestamp1 = \
jsonTopoMetrics1[ graphTimestamp ][ 'value' ]
graphTimestamp2 = \
jsonTopoMetrics2[ graphTimestamp ][ 'value' ]
graphTimestamp3 = \
jsonTopoMetrics3[ graphTimestamp ][ 'value' ]
linkTimestamp1 = \
jsonTopoMetrics1[ linkTimestamp ][ 'value' ]
linkTimestamp2 = \
jsonTopoMetrics2[ linkTimestamp ][ 'value' ]
linkTimestamp3 = \
jsonTopoMetrics3[ linkTimestamp ][ 'value' ]
if graphTimestamp1 and graphTimestamp2 and\
graphTimestamp3 and linkTimestamp1 and\
linkTimestamp2 and linkTimestamp3:
linkUpLatGraph1 = int( graphTimestamp1 ) -\
int( timestampLinkUpT0 )
linkUpLatGraph2 = int( graphTimestamp2 ) -\
int( timestampLinkUpT0 )
linkUpLatGraph3 = int( graphTimestamp3 ) -\
int( timestampLinkUpT0 )
linkUpLatLink1 = int( linkTimestamp1 ) -\
int( timestampLinkUpT0 )
linkUpLatLink2 = int( linkTimestamp2 ) -\
int( timestampLinkUpT0 )
linkUpLatLink3 = int( linkTimestamp3 ) -\
int( timestampLinkUpT0 )
else:
main.log.error( "There was an error calculating" +
" the delta for link down event" )
linkUpLatGraph1 = 0
linkUpLatGraph2 = 0
linkUpLatGraph3 = 0
linkUpLatDevice1 = 0
linkUpLatDevice2 = 0
linkUpLatDevice3 = 0
if debugMode == 'on':
main.log.info( "Link up latency ONOS1 iteration " +
str( i ) + " (end-to-end): " +
str( linkUpLatGraph1 ) + " ms" )
main.log.info( "Link up latency ONOS2 iteration " +
str( i ) + " (end-to-end): " +
str( linkUpLatGraph2 ) + " ms" )
main.log.info( "Link up latency ONOS3 iteration " +
str( i ) + " (end-to-end): " +
str( linkUpLatGraph3 ) + " ms" )
main.log.info(
"Link up latency ONOS1 iteration " +
str( i ) +
" (link-event-to-system-timestamp): " +
str( linkUpLatLink1 ) +
" ms" )
main.log.info(
"Link up latency ONOS2 iteration " +
str( i ) +
" (link-event-to-system-timestamp): " +
str( linkUpLatLink2 ) +
" ms" )
main.log.info(
"Link up latency ONOS3 iteration " +
str( i ) +
" (link-event-to-system-timestamp): " +
str( linkUpLatLink3 ) )
# Calculate avg of node calculations
linkUpLatGraphAvg =\
( linkUpLatGraph1 +
linkUpLatGraph2 +
linkUpLatGraph3 ) / 3
linkUpLatLinkAvg =\
( linkUpLatLink1 +
linkUpLatLink2 +
linkUpLatLink3 ) / 3
# Set threshold and append latency to list
if linkUpLatGraphAvg > upThresholdMin and\
linkUpLatGraphAvg < upThresholdMax:
linkUpGraphToSystemList.append(
linkUpLatGraphAvg )
else:
main.log.info( "Link up latency exceeded threshold" )
main.log.info( "Results for iteration " + str( i ) +
"have been omitted" )
if linkUpLatLinkAvg > upThresholdMin and\
linkUpLatLinkAvg < upThresholdMax:
linkUpLinkToSystemList.append(
linkUpLatLinkAvg )
else:
main.log.info( "Link up latency exceeded threshold" )
main.log.info( "Results for iteration " + str( i ) +
"have been omitted" )
# Calculate min, max, avg of list and report
linkDownMin = min( linkDownGraphToSystemList )
linkDownMax = max( linkDownGraphToSystemList )
linkDownAvg = sum( linkDownGraphToSystemList ) / \
len( linkDownGraphToSystemList )
linkUpMin = min( linkUpGraphToSystemList )
linkUpMax = max( linkUpGraphToSystemList )
linkUpAvg = sum( linkUpGraphToSystemList ) / \
len( linkUpGraphToSystemList )
linkDownStdDev = \
str( round( numpy.std( linkDownGraphToSystemList ), 1 ) )
linkUpStdDev = \
str( round( numpy.std( linkUpGraphToSystemList ), 1 ) )
main.log.report( "Link down latency " +
"Avg: " + str( linkDownAvg ) + " ms " +
"Std Deviation: " + linkDownStdDev + " ms" )
main.log.report( "Link up latency " +
"Avg: " + str( linkUpAvg ) + " ms " +
"Std Deviation: " + linkUpStdDev + " ms" )
utilities.assert_equals(
expect=main.TRUE,
actual=assertion,
onpass="Link discovery latency calculation successful",
onfail="Link discovery latency case failed" )
def CASE5( self, main ):
"""
100 Switch discovery latency
Important:
This test case can be potentially dangerous if
your machine has previously set iptables rules.
One of the steps of the test case will flush
all existing iptables rules.
Note:
You can specify the number of switches in the
params file to adjust the switch discovery size
( and specify the corresponding topology in Mininet1
.topo file )
"""
import time
import subprocess
import os
import requests
import json
ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
MN1Ip = main.params[ 'MN' ][ 'ip1' ]
ONOSUser = main.params[ 'CTRL' ][ 'user' ]
defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
# Number of iterations of case
numIter = main.params[ 'TEST' ][ 'numIter' ]
numSw = main.params[ 'TEST' ][ 'numSwitch' ]
# Timestamp 'keys' for json metrics output.
# These are subject to change, hence moved into params
deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
debugMode = main.params[ 'TEST' ][ 'debugMode' ]
localTime = time.strftime( '%X' )
localTime = localTime.replace( "/", "" )
localTime = localTime.replace( " ", "_" )
localTime = localTime.replace( ":", "" )
if debugMode == 'on':
main.ONOS1.tsharkPcap( "eth0",
"/tmp/100_sw_lat_pcap_" + localTime )
# Threshold for this test case
swDiscThresholdStr = main.params[ 'TEST' ][ 'swDisc100Threshold' ]
swDiscThresholdObj = swDiscThresholdStr.split( "," )
swDiscThresholdMin = int( swDiscThresholdObj[ 0 ] )
swDiscThresholdMax = int( swDiscThresholdObj[ 1 ] )
tsharkOfpOutput = "/tmp/tshark_ofp_" + numSw + "sw.txt"
tsharkTcpOutput = "/tmp/tshark_tcp_" + numSw + "sw.txt"
tsharkOfpResultList = []
tsharkTcpResultList = []
swDiscoveryLatList = []
main.case( numSw + " Switch discovery latency" )
main.step( "Assigning all switches to ONOS1" )
for i in range( 1, int( numSw ) + 1 ):
main.Mininet1.assignSwController(
sw=str( i ),
ip1=ONOS1Ip,
port1=defaultSwPort )
# Ensure that nodes are configured with ptpd
# Just a warning message
main.log.info( "Please check ptpd configuration to ensure" +
" All nodes' system times are in sync" )
time.sleep( 5 )
for i in range( 0, int( numIter ) ):
main.step( "Set iptables rule to block incoming sw connections" )
# Set iptables rule to block incoming switch connections
# The rule description is as follows:
# Append to INPUT rule,
# behavior DROP that matches following:
# * packet type: tcp
# * source IP: MN1Ip
# * destination PORT: 6633
main.ONOS1.handle.sendline(
"sudo iptables -A INPUT -p tcp -s " + MN1Ip +
" --dport " + defaultSwPort + " -j DROP" )
main.ONOS1.handle.expect( "\$" )
# Append to OUTPUT rule,
# behavior DROP that matches following:
# * packet type: tcp
# * source IP: MN1Ip
# * destination PORT: 6633
main.ONOS1.handle.sendline(
"sudo iptables -A OUTPUT -p tcp -s " + MN1Ip +
" --dport " + defaultSwPort + " -j DROP" )
main.ONOS1.handle.expect( "\$" )
# Give time to allow rule to take effect
# NOTE: Sleep period may need to be configured
# based on the number of switches in the topology
main.log.info( "Please wait for switch connection to " +
"time out" )
time.sleep( 60 )
# Gather vendor OFP with tshark
main.ONOS1.tsharkGrep( "OFP 86 Vendor",
tsharkOfpOutput )
main.ONOS1.tsharkGrep( "TCP 74 ",
tsharkTcpOutput )
# NOTE: Remove all iptables rule quickly ( flush )
# Before removal, obtain TestON timestamp at which
# removal took place
# ( ensuring nodes are configured via ptp )
# sudo iptables -F
t0System = time.time() * 1000
main.ONOS1.handle.sendline(
"sudo iptables -F" )
# Counter to track loop count
counterLoop = 0
counterAvail1 = 0
counterAvail2 = 0
counterAvail3 = 0
onos1Dev = False
onos2Dev = False
onos3Dev = False
while counterLoop < 60:
# Continue to check devices for all device
# availability. When all devices in all 3
# ONOS instances indicate that devices are available
# obtain graph event timestamp for t1.
deviceStrObj1 = main.ONOS1cli.devices()
deviceStrObj2 = main.ONOS2cli.devices()
deviceStrObj3 = main.ONOS3cli.devices()
deviceJson1 = json.loads( deviceStrObj1 )
deviceJson2 = json.loads( deviceStrObj2 )
deviceJson3 = json.loads( deviceStrObj3 )
for device1 in deviceJson1:
if device1[ 'available' ]:
counterAvail1 += 1
if counterAvail1 == int( numSw ):
onos1Dev = True
main.log.info( "All devices have been " +
"discovered on ONOS1" )
else:
counterAvail1 = 0
for device2 in deviceJson2:
if device2[ 'available' ]:
counterAvail2 += 1
if counterAvail2 == int( numSw ):
onos2Dev = True
main.log.info( "All devices have been " +
"discovered on ONOS2" )
else:
counterAvail2 = 0
for device3 in deviceJson3:
if device3[ 'available' ]:
counterAvail3 += 1
if counterAvail3 == int( numSw ):
onos3Dev = True
main.log.info( "All devices have been " +
"discovered on ONOS3" )
else:
counterAvail3 = 0
if onos1Dev and onos2Dev and onos3Dev:
main.log.info( "All devices have been discovered " +
"on all ONOS instances" )
jsonStrTopologyMetrics1 =\
main.ONOS1cli.topologyEventsMetrics()
jsonStrTopologyMetrics2 =\
main.ONOS2cli.topologyEventsMetrics()
jsonStrTopologyMetrics3 =\
main.ONOS3cli.topologyEventsMetrics()
# Exit while loop if all devices discovered
break
counterLoop += 1
# Give some time in between CLI calls
#( will not affect measurement )
time.sleep( 3 )
main.ONOS1.tsharkStop()
os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
tsharkOfpOutput + " /tmp/" )
os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
tsharkTcpOutput + " /tmp/" )
# TODO: Automate OFP output analysis
# Debug mode - print out packets captured at runtime
if debugMode == 'on':
ofpFile = open( tsharkOfpOutput, 'r' )
main.log.info( "Tshark OFP Vendor output: " )
for line in ofpFile:
tsharkOfpResultList.append( line )
main.log.info( line )
ofpFile.close()
tcpFile = open( tsharkTcpOutput, 'r' )
main.log.info( "Tshark TCP 74 output: " )
for line in tcpFile:
tsharkTcpResultList.append( line )
main.log.info( line )
tcpFile.close()
jsonObj1 = json.loads( jsonStrTopologyMetrics1 )
jsonObj2 = json.loads( jsonStrTopologyMetrics2 )
jsonObj3 = json.loads( jsonStrTopologyMetrics3 )
graphTimestamp1 = \
jsonObj1[ graphTimestamp ][ 'value' ]
graphTimestamp2 = \
jsonObj2[ graphTimestamp ][ 'value' ]
graphTimestamp3 = \
jsonObj3[ graphTimestamp ][ 'value' ]
graphLat1 = int( graphTimestamp1 ) - int( t0System )
graphLat2 = int( graphTimestamp2 ) - int( t0System )
graphLat3 = int( graphTimestamp3 ) - int( t0System )
avgGraphLat = \
( int( graphLat1 ) +
int( graphLat2 ) +
int( graphLat3 ) ) / 3
if avgGraphLat > swDiscThresholdMin \
and avgGraphLat < swDiscThresholdMax:
swDiscoveryLatList.append(
avgGraphLat )
else:
main.log.info( "100 Switch discovery latency " +
"exceeded the threshold." )
# END ITERATION FOR LOOP
swLatMin = min( swDiscoveryLatList )
swLatMax = max( swDiscoveryLatList )
swLatAvg = sum( swDiscoveryLatList ) /\
len( swDiscoveryLatList )
main.log.report( "100 Switch discovery lat " +
"Min: " + str( swLatMin ) + " ms" +
"Max: " + str( swLatMax ) + " ms" +
"Avg: " + str( swLatAvg ) + " ms" )
def CASE6( self, main ):
"""
Increase number of nodes and initiate CLI
"""
import time
ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
cellName = main.params[ 'ENV' ][ 'cellName' ]
global clusterCount
# Cluster size increased everytime the case is defined
clusterCount += 2
main.log.report( "Increasing cluster size to " +
str( clusterCount ) )
installResult = main.FALSE
if clusterCount == 3:
main.log.info( "Installing nodes 2 and 3" )
node2Result = \
main.ONOSbench.onosInstall( node=ONOS2Ip )
node3Result = \
main.ONOSbench.onosInstall( node=ONOS3Ip )
installResult = node2Result and node3Result
time.sleep( 5 )
main.ONOS2cli.startOnosCli( ONOS2Ip )
main.ONOS3cli.startOnosCli( ONOS3Ip )
elif clusterCount == 5:
main.log.info( "Installing nodes 4 and 5" )
node4Result = \
main.ONOSbench.onosInstall( node=ONOS4Ip )
node5Result = \
main.ONOSbench.onosInstall( node=ONOS5Ip )
installResult = node4Result and node5Result
time.sleep( 5 )
main.ONOS4cli.startOnosCli( ONOS4Ip )
main.ONOS5cli.startOnosCli( ONOS5Ip )
elif clusterCount == 7:
main.log.info( "Installing nodes 4 and 5" )
node6Result = \
main.ONOSbench.onosInstall( node=ONOS6Ip )
node7Result = \
main.ONOSbench.onosInstall( node=ONOS7Ip )
installResult = node6Result and node7Result
time.sleep( 5 )
main.ONOS6cli.startOnosCli( ONOS6Ip )
main.ONOS7cli.startOnosCli( ONOS7Ip )