[SDFAB-284] Paired Leaves Upstream link failure test
TODO: TO test performance, we need to choose a target not on the
internet. Currently just pinging 8.8.8.8
Change-Id: I164f6bb44d6fa9685c7a466a8a0a83a5926ac395
diff --git a/TestON/drivers/common/cli/hostdriver.py b/TestON/drivers/common/cli/hostdriver.py
index 7111c74..080f6cc 100644
--- a/TestON/drivers/common/cli/hostdriver.py
+++ b/TestON/drivers/common/cli/hostdriver.py
@@ -39,6 +39,7 @@
self.inband = False
self.prompt = "\$"
self.scapyPrompt = ">>>"
+ self.tempRoutes = []
def connect( self, **connectargs ):
"""
@@ -120,6 +121,9 @@
try:
if self.handle:
self.preDisconnect()
+ if self.tempRoutes:
+ for r in self.tempRoutes:
+ self.deleteRoute( *r )
# Disconnect from the host
if not self.options[ 'inband' ] == 'True':
self.handle.sendline( "" )
@@ -219,6 +223,132 @@
main.log.error( self.name + ": " + self.handle.before )
return main.FALSE
+ def getRoutes( self ):
+ """
+ Returns the output of 'ip route show' command
+ # TODO: process and return a a list or json object
+ """
+ try:
+ cmd = "ip route show"
+ self.handle.sendline( cmd )
+ i = self.handle.expect( [ "password|Password", self.prompt ] )
+ if i == 0:
+ self.handle.sendline( self.pwd )
+ j = self.handle.expect( [ "password|Password", self.prompt ] )
+ if j != 1:
+ main.log.error( "Incorrect password" )
+ return main.FALSE
+ return self.handle.before
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+
+ def addRoute( self, route, gw, interface=None, sudoRequired=True, purgeOnDisconnect=True ):
+ """
+ Adds a static route to the host
+ Arguments:
+ * route - String, CIDR notation for the destination subnet
+ * gw - String, IP address of gateway
+ Optional Arguments:
+ * interface - String, The interface to use for this route, defaults to None, or the default interface
+ * sudoRequired - Boolean, whether sudo is needed for this command, defaults to True
+ * purgeOnDisconnect - Boolean, remove this route before disconnecting from component
+ """
+ try:
+ cmd = "ip route add %s via %s" % ( route, gw )
+ if sudoRequired:
+ cmd = "sudo %s" % cmd
+ if interface:
+ cmd = "%s dev %s" % ( cmd, interface )
+ if purgeOnDisconnect:
+ self.tempRoutes.append( (route, gw, interface, sudoRequired ) )
+ self.handle.sendline( cmd )
+ i = self.handle.expect( [ "password|Password", self.prompt, "Error" ] )
+ output = ""
+ if i == 2:
+ output += self.handle.before + self.handle.after
+ self.handle.expect( self.prompt )
+ output += self.handle.before + self.handle.after
+ main.log.error( "%s: Erorr in ip route command: %s" % ( self.name, output ) )
+ return main.FALSE
+ elif i == 0:
+ self.handle.sendline( self.pwd )
+ j = self.handle.expect( [ "password|Password", self.prompt, "Error" ] )
+ if j == 0:
+ main.log.error( "Incorrect password" )
+ return main.FALSE
+ elif j == 2:
+ output += self.handle.before + self.handle.after
+ self.handle.expect( self.prompt )
+ output += self.handle.before + self.handle.after
+ main.log.error( "%s: Erorr in ip route command: %s" % ( self.name, output ) )
+ return main.FALSE
+ output += self.handle.before + self.handle.after
+ main.log.debug( output )
+ return self.handle.before
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+
+ def deleteRoute( self, route, gw, interface=None, sudoRequired=True ):
+ """
+ Deletess a static route from the host
+ Arguments:
+ * route - String, CIDR notation for the destination subnet
+ * gw - String, IP address of gateway
+ Optional Arguments:
+ * interface - String, The interface to use for this route, defaults to None, or the default interface
+ * sudoRequired - Boolean, whether sudo is needed for this command, defaults to True
+ """
+ try:
+ cmd = "ip route del %s via %s" % ( route, gw )
+ if sudoRequired:
+ cmd = "sudo %s" % cmd
+ if interface:
+ cmd = "%s dev %s" % ( cmd, interface )
+ self.handle.sendline( cmd )
+ i = self.handle.expect( [ "password|Password", self.prompt, "Error" ] )
+ output = ""
+ if i == 2:
+ output += self.handle.before + self.handle.after
+ self.handle.expect( self.prompt )
+ output += self.handle.before + self.handle.after
+ main.log.error( "%s: Erorr in ip route command: %s" % ( self.name, output ) )
+ return main.FALSE
+ elif i == 0:
+ self.handle.sendline( self.pwd )
+ j = self.handle.expect( [ "password|Password", self.prompt, "Error" ] )
+ if j == 0:
+ main.log.error( "Incorrect password" )
+ return main.FALSE
+ elif j == 2:
+ output += self.handle.before + self.handle.after
+ self.handle.expect( self.prompt )
+ output += self.handle.before + self.handle.after
+ main.log.error( "%s: Erorr in ip route command: %s" % ( self.name, output ) )
+ return main.FALSE
+ output += self.handle.before + self.handle.after
+ main.log.debug( output )
+ return self.handle.before
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+
def getIPAddress( self, iface=None, proto='IPV4' ):
"""
Returns IP address of the host
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 7c2acb5..bafa284 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -6908,3 +6908,85 @@
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+
+ def routeAdd( self, prefix, nextHop, jsonFormat=True ):
+ """
+ Add a route with the given prefix and nextHop
+ """
+ try:
+ cmdStr = "route-add %s %s %s" % ( "-j" if jsonFormat else "", prefix, nextHop )
+ handle = self.sendline( cmdStr )
+ assert handle is not None, "Error in sendline"
+ assert "Command not found:" not in handle, handle
+ assert "Unsupported command:" not in handle, handle
+ assert "Error executing command" not in handle, handle
+ return handle
+ except AssertionError:
+ main.log.exception( "" )
+ return None
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
+ def routeRemove( self, prefix, nextHop, source=None, jsonFormat=True ):
+ """
+ Remove a route with the given prefix and nextHop
+ """
+ try:
+ if source:
+ raise NotImplemented
+ cmdStr = "route-remove %s %s %s" % ( "-j" if jsonFormat else "", prefix, nextHop )
+ handle = self.sendline( cmdStr )
+ assert handle is not None, "Error in sendline"
+ assert "Command not found:" not in handle, handle
+ assert "Unsupported command:" not in handle, handle
+ assert "Error executing command" not in handle, handle
+ return handle
+ except AssertionError:
+ main.log.exception( "" )
+ return None
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
+ def routes( self, jsonFormat=True, args="" ):
+ """
+ Get routes from ONOS
+ """
+ try:
+ cmdStr = "routes"
+ if jsonFormat:
+ cmdStr = cmdStr + " -j"
+ handle = self.sendline( cmdStr )
+ assert handle is not None, "Error in sendline"
+ assert "Command not found:" not in handle, handle
+ assert "Unsupported command:" not in handle, handle
+ assert "Error executing command" not in handle, handle
+ return handle
+ except AssertionError:
+ main.log.exception( "" )
+ return None
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
index da1157a..2cb31b3 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,101,102,103,104,201,202,203,204,205,206,207,208</testcases>
+ <testcases>1,2,101,102,103,104,201,202,203,204,205,206,207,208,301</testcases>
<GRAPH>
<nodeCluster>pairedleaves</nodeCluster>
@@ -78,7 +78,7 @@
<timers>
<LinkDiscovery>12</LinkDiscovery>
<SwitchDiscovery>12</SwitchDiscovery>
- <TrafficDiscovery>10</TrafficDiscovery>
+ <TrafficDiscovery>13</TrafficDiscovery>
</timers>
<SLEEP>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
index 3422244..71cc5c2 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
@@ -565,3 +565,108 @@
# Cleanup
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
main.funcs.cleanup( main )
+
+ def CASE301( self, main ):
+ """
+ Connect to Pod
+ Setup static route to public internet via mgmt
+ Ping public ip
+ chech which link the ICMP traffic is going to
+ Kill that link
+ Verify flow continues using other link
+ Restore link
+ verify traffic still flows
+ Collect logs and analyze results
+ """
+ try:
+ from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
+ import re
+ except ImportError:
+ main.log.error( "SRStagingTest not found. Exiting the test" )
+ main.cleanAndExit()
+ try:
+ main.funcs
+ except ( NameError, AttributeError ):
+ main.funcs = SRStagingTest()
+
+ descPrefix = "CASE301-Upstream-Link"
+ pod = main.params['GRAPH'].get( 'nodeCluster', "hardware" )
+ main.funcs.setupTest( main,
+ topology='0x2',
+ onosNodes=3,
+ description="%s tests on the %s pod" % ( descPrefix, pod ) )
+
+ # TODO Route ADD
+ dstIp = "8.8.8.8"
+ route = "%s/32" % dstIp
+ """
+ Try this on the host:
+ ip route add 8.8.8.8/32 via <fabric interface ip>
+ and this in ONOS:
+ route-add 8.8.8.8/32 via <mgmt server fabric ip
+ """
+
+ srcComponent = getattr( main, 'Compute1' )
+ nextHopComponent = getattr( main, 'ManagmentServer' )
+
+ # Add route in host to outside host via gateway ip
+ srcIface = srcComponent.interfaces[0].get( 'name' )
+ srcIp = srcComponent.getIPAddress( iface=srcIface )
+ hostsJson = json.loads( main.Cluster.active( 0 ).hosts() )
+ netcfgJson = json.loads( main.Cluster.active( 0 ).getNetCfg( subjectClass='ports') )
+ ips = []
+ fabricIntfIp = None
+ for obj in hostsJson:
+ if srcIp in obj['ipAddresses']:
+ for location in obj['locations']:
+ main.log.debug( location )
+ did = location['elementId'].encode( 'utf-8' )
+ port = location['port'].encode( 'utf-8' )
+ m = re.search( '\((\d+)\)', port )
+ if m:
+ port = m.group(1)
+ portId = "%s/%s" % ( did, port )
+ # Lookup ip assigned to this network port
+ ips.extend( [ x.encode( 'utf-8' ) for x in netcfgJson[ portId ][ 'interfaces' ][0][ 'ips' ] ] )
+ ips = set( ips )
+ ipRE = r'(\d+\.\d+\.\d+\.\d+)/\d+|([\w,:]*)/\d+'
+ for ip in ips:
+ ipMatch = re.search( ipRE, ip )
+ if ipMatch:
+ fabricIntfIp = ipMatch.group(1)
+ main.log.debug( "Found %s as gateway ip for %s" % ( fabricIntfIp, srcComponent.shortName ) )
+ # FIXME: How to chose the correct one if there are multiple? look at subnets
+ srcComponent.addRoute( route, fabricIntfIp, srcIface, sudoRequired=True, purgeOnDisconnect=True )
+ main.log.debug( srcComponent.getRoutes() )
+
+ # Add route in ONOS
+ nextHopIface = nextHopComponent.interfaces[0].get( 'name' )
+ nextHopIp = nextHopComponent.getIPAddress( iface=nextHopIface )
+ main.Cluster.active( 0 ).routeAdd( route, nextHopIp )
+ main.log.debug( main.Cluster.active( 0 ).routes() )
+
+
+
+ ####
+ targets = main.funcs.getHostConnections( main, nextHopComponent )
+ shortDesc = descPrefix + "-Failure"
+ longDesc = "%s Failure: Bring down port with traffic to %s" % ( descPrefix, route )
+ #TODO: Option to just do ping
+ killDevice, killPort = main.funcs.linkDown( targets, srcComponent, nextHopComponent, shortDesc,
+ longDesc, stat='packetsSent', bidirectional=False,
+ pingOnly=True, dstIp=dstIp )
+ shortDesc = descPrefix + "-Recovery"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, killDevice, killPort )
+ main.funcs.linkUp( killDevice, killPort, srcComponent, nextHopComponent, shortDesc, longDesc,
+ bidirectional=False, pingOnly=True, dstIp=dstIp )
+
+ # Remove route in ONOS
+ main.Cluster.active( 0 ).routeRemove( route, nextHopIp )
+ main.log.debug( main.Cluster.active( 0 ).routes() )
+ # Remove route on host
+ srcComponent.deleteRoute( route, fabricIntfIp, srcIface, sudoRequired=True )
+ main.log.debug( srcComponent.getRoutes() )
+ # Cleanup
+ main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+ main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index 61ae1a1..7b87198 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -160,51 +160,72 @@
main.log.warn( "%s: %s" % ( src.name, str( src.handle.before ) ) )
@staticmethod
- def startIperf( main, src, dst, trafficSelector, trafficDuration ):
+ def startIperf( main, src, dstIp, trafficSelector, trafficDuration ):
"""
Start iperf from src ip to dst ip
Handles connectivity check and sudo password input as well
Arguments:
src: src host component
- dst: dst host component
+ dstIp: dst ip
trafficSelector: traffic selector string, passed to iperf
trafficDuration: Traffic duration string, passed to iperf
"""
- dstIp = dst.interfaces[0]['ips'][0]
srcIp = src.interfaces[0]['ips'][0]
iperfArgs = "%s --bind %s -c %s -t %s" % ( trafficSelector,
srcIp,
dstIp,
trafficDuration )
- main.log.info( "Starting iperf between %s and %s" % ( src.shortName, dst.shortName ) )
- sudoCheck = main.funcs.singlePingWithSudo( main, src, dst.interfaces[0]['ips'][0] )
+ main.log.info( "Starting iperf between %s and %s" % ( src.shortName, dstIp ) )
+ sudoCheck = main.funcs.singlePingWithSudo( main, src, dstIp )
src.handle.sendline( "/usr/bin/iperf %s " % iperfArgs )
src.preDisconnect = src.exitFromProcess
@staticmethod
- def startTshark( main, host, pingDesc=None, direction="Sender", srcIP=None, dstIP=None ):
+ def startPing( main, src, dstIp, count=None, interval=".3" ):
+ """
+ Start ping from src to dst ip
+ Arguments:
+ src: src host component
+ dstIp: dst ip
+ count: how many packets to send, defaults to None, or until stopped
+ interval: How long to wait, in seconds, between pings, defaults to .5
+ """
+ srcIface = src.interfaces[0]['name']
+ pingCmd = "ping -I %s %s " % ( srcIface, dstIp )
+ if count:
+ pingCmd += " -c %s " % count
+ if interval:
+ pingCmd += " -i %s " % interval
+
+ main.log.info( "Starting ping between %s and %s" % ( src.shortName, dstIp ) )
+ src.handle.sendline( pingCmd )
+ src.preDisconnect = src.exitFromProcess
+
+
+ @staticmethod
+ def startTshark( main, host, pingDesc=None, direction="Sender", srcIp=None, dstIp=None, protocolStr="udp" ):
"""
"""
+ hostStr = ""
if direction == "Sender":
suffix = "Sender"
- hostStr = "src host %s" % host.interfaces[0]['ips'][0]
- if dstIP:
- hostStr += " && dst host %s" % dstIP
else:
suffix = "Receiver"
- hostStr = "dst host %s" % host.interfaces[0]['ips'][0]
- if srcIP:
- hostStr += " && src host %s" % srcIP
+ if dstIp:
+ hostStr += " && dst host %s" % dstIp
+ if srcIp:
+ hostStr += " && src host %s" % srcIp
if not pingDesc:
pingDesc = host.name
fileName = "%s-tshark%s" % ( pingDesc, suffix )
pcapFile = "%s/tshark/%s.pcap" % ( "~/TestON", fileName )
- tsharkArgs = "%s -i %s -f 'udp && %s' -w %s" % ( main.params[ 'PERF' ][ 'pcap_cmd_arguments' ],
- host.interfaces[0]['name'],
- hostStr,
- pcapFile )
+ tsharkArgs = "%s -i %s -f '%s %s' -w %s" % ( main.params[ 'PERF' ][ 'pcap_cmd_arguments' ],
+ host.interfaces[0]['name'],
+ protocolStr,
+ hostStr,
+ pcapFile )
commands = [ 'mkdir -p ~/TestON/tshark',
'rm %s' % pcapFile,
'touch %s' % pcapFile,
@@ -219,26 +240,41 @@
@staticmethod
def setupFlow( main, src, dst, shortDesc=None, longDesc=None,
- trafficDuration=600, trafficSelector="-u -b 20M" ):
+ trafficDuration=600, trafficSelector="-u -b 20M", pingOnly=False, dstIp=None ):
"""
Setup iperf flow between two hosts, also handles packet captures, etc.
"""
try:
- main.log.info( "Setting up flow between %s and %s" % ( src.shortName, dst.shortName ) )
+ if not dstIp:
+ dstIp = dst.interfaces[0]['ips'][0]
+ main.log.info( "Setting up flow between %s and %s%s" % ( src.shortName, dst.shortName, "" if not dstIp else " with dstIp %s" % dstIp ) )
# ping right before to make sure arp is cached and sudo is authenticated
main.funcs.singlePingWithSudo( main, src, dst.interfaces[0]['ips'][0] )
main.funcs.singlePingWithSudo( main, dst, src.interfaces[0]['ips'][0] )
# Start traffic
# TODO: ASSERTS
- iperfSrc = getattr( main, "%s-iperf" % src.name )
- main.funcs.startIperf( main, iperfSrc, dst, trafficSelector, trafficDuration )
+ if pingOnly:
+ trafficCmd = "ping"
+ protocolStr = "icmp"
+ else:
+ trafficCmd = "iperf"
+ protocolStr = "udp"
+ trafficSrc = getattr( main, "%s-%s" % ( src.name, trafficCmd ) )
+ if trafficCmd == "iperf":
+ main.funcs.startIperf( main, trafficSrc, dstIp, trafficSelector, trafficDuration )
+ elif trafficCmd == "ping":
+ main.funcs.startPing( main, trafficSrc, dstIp )
+ else:
+ raise NotImplemented, "Unknown Traffic Command"
# Start packet capture
pingDesc = "%s-%s-to-%s" % ( shortDesc, src.shortName, dst.shortName )
pcapFileReceiver = "%s/tshark/%s-tsharkReceiver.pcap" % ( "~/TestON", pingDesc )
pcapFileSender = "%s/tshark/%s-tsharkSender.pcap" % ( "~/TestON", pingDesc )
- # FIXME: Also filter by sender here
- main.funcs.startTshark( main, dst, pingDesc=pingDesc, direction="Receiver", srcIP=src.interfaces[0]['ips'][0] )
- main.funcs.startTshark( main, src, pingDesc=pingDesc, direction="Sender", dstIP=dst.interfaces[0]['ips'][0] )
+ srcIp = src.interfaces[0]['ips'][0]
+ main.funcs.startTshark( main, dst, pingDesc=pingDesc, direction="Receiver",
+ srcIp=srcIp, dstIp=dstIp, protocolStr=protocolStr )
+ main.funcs.startTshark( main, src, pingDesc=pingDesc, direction="Sender",
+ srcIp=srcIp, dstIp=dstIp, protocolStr=protocolStr )
except Exception as e:
main.log.exception( "Error in setupFlow" )
@@ -248,7 +284,7 @@
def startCapturing( main, srcList, dstList, shortDesc=None, longDesc=None,
trafficDuration=600, trafficSelector="-u -b 20M",
bidirectional=False, singleFlow=False, targets=None,
- threshold=10, stat="packetsSent" ):
+ threshold=10, stat="packetsSent", pingOnly=False, dstIp=None ):
"""
Starts logging, traffic generation, traffic filters, etc before a failure is induced
src: the src component that sends the traffic
@@ -300,6 +336,7 @@
if not isinstance( dstList, list ):
dstList = [ dstList ]
hostPairs = []
+ trafficCmd = "ping" if pingOnly else "iperf"
for src in srcList:
for dst in dstList:
if src == dst:
@@ -315,8 +352,8 @@
receiver = getattr( main, receiverName )
main.trafficComponents.append( receiver )
hostPairs.append( ( sender, receiver ) )
- main.Network.copyComponent( src.name, "%s-iperf" % senderName )
- main.trafficComponents.append( getattr( main, "%s-iperf" % senderName ) )
+ main.Network.copyComponent( src.name, "%s-%s" % ( senderName, trafficCmd ) )
+ main.trafficComponents.append( getattr( main, "%s-%s" % ( senderName, trafficCmd ) ) )
newName = "%s-%s" % ( dst.shortName, "FileSize" )
main.Network.copyComponent( dst.name, newName )
main.trafficComponents.append( getattr( main, newName ) )
@@ -335,8 +372,8 @@
receiver = getattr( main, receiverName )
main.trafficComponents.append( receiver )
hostPairs.append( ( sender, receiver ) )
- main.Network.copyComponent( dst.name, "%s-iperf" % senderName )
- main.trafficComponents.append( "%s-iperf" % senderName )
+ main.Network.copyComponent( dst.name, "%s-%s" % ( senderName, trafficCmd ) )
+ main.trafficComponents.append( "%s-%s" % ( senderName, trafficCmd ) )
# TODO: make sure hostPairs is a set?
main.log.debug( ["%s to %s" % ( p[0], p[1] ) for p in hostPairs ] )
@@ -348,10 +385,10 @@
src, dst = pair
main.funcs.setupFlow( main, src, dst, shortDesc=shortDesc,
longDesc=longDesc, trafficDuration=trafficDuration,
- trafficSelector=trafficSelector )
+ trafficSelector=trafficSelector, pingOnly=pingOnly, dstIp=dstIp )
if singleFlow:
# Let some packets flow
- trafficSleep = float( main.params['timers'].get( 'TrafficDiscovery', 5 ) )
+ trafficSleep = float( main.params['timers'].get( 'TrafficDiscovery', 10 ) )
main.log.info( "Sleeping %s seconds for packet counters to update" % trafficSleep )
time.sleep( trafficSleep )
updatedStats = json.loads( main.Cluster.active(0).REST.portstats() )
@@ -366,12 +403,13 @@
break
else:
main.funcs.stopFlow( main, src, dst, shortDesc=shortDesc,
- longDesc=longDesc, abort=True )
+ longDesc=longDesc, abort=True,
+ pingOnly=pingOnly )
if singleFlow and not switchComponent:
main.log.error( "Could not find a flow going through desired switch/port, aborting test" )
main.skipCase( result="PASS" )
main.pingStarted = time.time()
- main.log.warn( "It took %s seconds to start all iperf and tshark sessions" % ( main.pingStarted - main.pingStart ) )
+ main.log.warn( "It took %s seconds to start all traffic and tshark sessions" % ( main.pingStarted - main.pingStart ) )
# Timestamp used for EVENT START
main.eventStart = datetime.datetime.utcnow()
@@ -396,7 +434,7 @@
return result
@staticmethod
- def stopFlow( main, src, dst, shortDesc=None, longDesc=None, abort=False ):
+ def stopFlow( main, src, dst, shortDesc=None, longDesc=None, abort=False, pingOnly=False ):
"""
Check flow is still connected, Stop iperf, tshark, etc
"""
@@ -438,12 +476,19 @@
main.funcs.clearBuffer( dst, kill=True, debug=True )
main.funcs.clearBuffer( src, kill=True, debug=True )
# Stop traffic
- iperfSrc = getattr( main, "%s-iperf" % src.name )
- main.funcs.clearBuffer( iperfSrc, kill=True, debug=True )
+ if pingOnly:
+ trafficSrc = getattr( main, "%s-ping" % src.name )
+ else:
+ trafficSrc = getattr( main, "%s-iperf" % src.name )
+ main.funcs.clearBuffer( trafficSrc, kill=True, debug=True )
if not abort:
srcIp = src.interfaces[0]['ips'][0]
- filterStr = "'udp && ip.src == %s'" % srcIp
+ if pingOnly:
+ filterStr = "'icmp "
+ else:
+ filterStr = "'udp "
+ filterStr += " && ip.src == %s'" % srcIp
#senderTime = main.funcs.analyzePcap( main, src, pcapFileSender, filterStr, debug=False )
#receiverTime = main.funcs.analyzePcap( main, dst, pcapFileReceiver, filterStr, debug=False )
#main.downtimeResults[ "%s" % senderResultDesc ] = senderTime # Orig
@@ -462,8 +507,14 @@
onpass="Saved pcap files from %s" % dst.name,
onfail="Failed to scp pcap files from %s" % dst.name )
- senderLosses = main.funcs.analyzeIperfPcap( main, main.logdir + "/" + pingDesc + "-tsharkSender.pcap", filterStr )
- receiverLosses = main.funcs.analyzeIperfPcap( main, main.logdir + "/" + pingDesc + "-tsharkReceiver.pcap", filterStr )
+ senderLosses = main.funcs.analyzeIperfPcap( main,
+ main.logdir + "/" + pingDesc + "-tsharkSender.pcap",
+ filterStr,
+ pingOnly=pingOnly )
+ receiverLosses = main.funcs.analyzeIperfPcap( main,
+ main.logdir + "/" + pingDesc + "-tsharkReceiver.pcap",
+ filterStr,
+ pingOnly=pingOnly )
ms, dropped = max( senderLosses, key=lambda i: i[0] )
colName = "%s" % senderResultDesc
main.downtimeResults[ colName[:63] ] = ms
@@ -481,7 +532,7 @@
@staticmethod
def stopCapturing( main, srcList, dstList, shortDesc=None, longDesc=None, bidirectional=False,
- killedNodes=None ):
+ killedNodes=None, pingOnly=False ):
import datetime
import time
from tests.dependencies.utils import Utils
@@ -523,7 +574,7 @@
for pair in hostPairs:
src, dst = pair
main.funcs.stopFlow( main, src, dst, shortDesc=shortDesc,
- longDesc=longDesc )
+ longDesc=longDesc, pingOnly=pingOnly )
main.pingStop = time.time()
main.log.warn( "It took %s seconds since we started to stop ping for us to stop pings" % ( main.pingStop - main.pingStopping ) )
@@ -555,6 +606,7 @@
killedPods.extend( pods )
switches = [ switch for switch in switches if switch not in killedPods ]
+ """
for switch in switches:
dstFile = "%s/%s-%s-write-reqs.txt" % ( main.logdir, shortDesc, switch )
writeResult = main.ONOSbench.kubectlCp( switch, "/tmp/p4_writes.txt", dstFile,
@@ -563,6 +615,7 @@
utilities.assert_equals( expect=main.TRUE, actual=writeResult,
onpass="Saved write-req file from %s" % switch,
onfail="Failed to cp write-req file from %s" % switch )
+ """
except Exception:
main.log.exception( "Error in stopCapturing" )
@@ -624,7 +677,8 @@
@staticmethod
def linkDown( targets, srcComponentList, dstComponentList, shortDesc,
- longDesc, sleepTime=10, stat='packetsSent', bidirectional=False ):
+ longDesc, sleepTime=10, stat='packetsSent', bidirectional=False,
+ pingOnly=False, dstIp=None ):
""""
High level function that handles an event including monitoring
Arguments:
@@ -638,6 +692,8 @@
sleepTime - How long to wait between starting the capture and stopping
stat - String, The stat to compare for each port across updates. Defaults to 'packetsSent'
bidirectional - Boolean, Whether to start traffic flows in both directions. Defaults to False
+ pingOnly - Boolean, Use ping if True else use iperf, defaults to False
+ - String, If set, use this as the destination IP for traffic, defaults to None
Returns:
A string of the port id that was brought down
"""
@@ -645,13 +701,16 @@
try:
initialStats = json.loads( main.Cluster.active(0).REST.portstats() )
main.step( "Start Capturing" )
+ threshold = 2 if pingOnly else 100
main.funcs.startCapturing( main,
srcComponentList,
dstComponentList,
shortDesc=shortDesc,
longDesc=longDesc,
bidirectional=bidirectional,
- threshold=100 )
+ threshold=threshold,
+ pingOnly=pingOnly,
+ dstIp=dstIp )
# Let some packets flow
trafficDiscoverySleep = float( main.params['timers'].get( 'TrafficDiscovery', 5 ) )
main.log.debug( "Sleeping %d seconds for traffic counters to update" % trafficDiscoverySleep )
@@ -680,7 +739,8 @@
dstComponentList,
shortDesc=shortDesc,
longDesc=longDesc,
- bidirectional=bidirectional )
+ bidirectional=bidirectional,
+ pingOnly=pingOnly )
# Break down logs
main.log.warn( main.logdir )
# This is not currently working, disabling for now
@@ -691,7 +751,7 @@
@staticmethod
def linkUp( device, port, srcComponentList, dstComponentList, shortDesc, longDesc,
- sleepTime=10, bidirectional=False ):
+ sleepTime=10, bidirectional=False, pingOnly=False, dstIp=None ):
""""
High level function that handles an event including monitoring
Arguments:
@@ -704,6 +764,8 @@
Option Arguments:
sleepTime - How long to wait between starting the capture and stopping
bidirectional - Boolean, Whether to start traffic flows in both directions. Defaults to False
+ pingOnly - Boolean, Use ping if True else use iperf, defaults to False
+ dstIp - String, If set, use this as the destination IP for traffic, defaults to None
"""
import time
if port is None:
@@ -717,7 +779,9 @@
shortDesc=shortDesc,
longDesc=longDesc,
bidirectional=bidirectional,
- threshold=100 )
+ threshold=100,
+ pingOnly=pingOnly,
+ dstIp=dstIp )
main.step( "Port Up" )
ctrl = main.Cluster.active( 0 ).CLI
portUp = ctrl.portstate( dpid=device, port=port, state="enable" )
@@ -738,7 +802,8 @@
dstComponentList,
shortDesc=shortDesc,
longDesc=longDesc,
- bidirectional=bidirectional )
+ bidirectional=bidirectional,
+ pingOnly=pingOnly )
# Break down logs
# This is not currently working, disabling for now
# main.funcs.analyzeLogs( shortDesc, 'portstate_up', main.eventStart, main.eventStop, main.logdir )
@@ -987,7 +1052,7 @@
main.log.exception( "Error in onosDown" )
@staticmethod
- def analyzeIperfPcap( main, pcapFile, filterStr, timeout=240 ):
+ def analyzeIperfPcap( main, pcapFile, filterStr, timeout=240, pingOnly=False ):
"""
Given a pcap file, will use tshark to create a csv file with iperf fields.
Then reads the file and computes drops in packets and duration of disruption
@@ -997,7 +1062,12 @@
import datetime
baseName = pcapFile[ :pcapFile.rfind('.') ]
csvFile = baseName + ".csv" # TODO: Strip any file extensions from pcapFile first
- tsharkCmd = 'tshark -r %s -Y %s -T fields -e frame.number -e frame.time_delta -e frame.time_epoch -e ip.src -e ip.dst -e iperf2.udp.sequence -d udp.port==5001,iperf2 -E separator=,' % ( pcapFile, filterStr )
+ tsharkCmd = 'tshark -r %s -Y %s -T fields -e frame.number -e frame.time_delta -e frame.time_epoch -e ip.src -e ip.dst ' % ( pcapFile, filterStr )
+ if pingOnly:
+ tsharkCmd += ' -e icmp.seq '
+ else:
+ tsharkCmd += ' -e iperf2.udp.sequence -d udp.port==5001,iperf2'
+ tsharkCmd += ' -E separator=,'
bench = main.ONOSbench
bench.handle.sendline( "%s > %s" % ( tsharkCmd, csvFile ) )
bench.handle.expect( bench.Prompt(), timeout=timeout )