Fix T3 for SRRouting
- Fix the issue that t3-simple doesn't work for external hosts
- Run t3 with both locations for dual-homed hosts
- Add timeout when dumping onos command to file
Change-Id: I2f93fad5c7d95d08f4b8703ecd606a7f070b4fa4
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 4c23ec1..7aa8a40 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -6513,7 +6513,7 @@
def composeT3Command( self, sAddr, dAddr, ipv6=False, verbose=True, simple=False ):
"""
- Compose and return t3-troubleshoot cli command for given source and destination addresses
+ Compose and return a list of t3-troubleshoot cli commands for given source and destination addresses
Options:
sAddr: IP address of the source host
dAddr: IP address of the destination host
@@ -6535,29 +6535,35 @@
if sHost and dHost:
break
assert sHost, "Not able to find host with IP {}".format( sAddr )
+ cmdList = []
if simple:
assert dHost, "Not able to find host with IP {}".format( dAddr )
- cmdStr = "t3-troubleshoot"
- if simple:
- cmdStr += "-simple"
- if verbose:
- cmdStr += " -vv"
- if ipv6:
- cmdStr += " -et ipv6"
- if simple:
+ cmdStr = "t3-troubleshoot-simple"
+ if verbose:
+ cmdStr += " -vv"
+ if ipv6:
+ cmdStr += " -et ipv6"
cmdStr += " {}/{} {}/{}".format( sHost[ "mac" ], sHost[ "vlan" ], dHost[ "mac" ], dHost[ "vlan" ] )
+ cmdList.append( cmdStr )
else:
- cmdStr += " -s " + str( sAddr )
- cmdStr += " -sp " + str( sHost[ "locations" ][ 0 ][ "elementId" ] ) + "/" + str( sHost[ "locations" ][ 0 ][ "port" ] )
- cmdStr += " -sm " + str( sHost[ "mac" ] )
- if sHost[ "vlan" ] != "None":
- cmdStr += " -vid " + sHost[ "vlan" ]
- cmdStr += " -d " + str( dAddr )
- netcfg = self.netcfg( args="devices {}".format( sHost[ "locations" ][ 0 ][ "elementId" ] ) )
- netcfg = json.loads( netcfg )
- assert netcfg, "Failed to get netcfg"
- cmdStr += " -dm " + str( netcfg[ "segmentrouting" ][ "routerMac" ] )
- return cmdStr
+ for location in sHost[ "locations" ]:
+ cmdStr = "t3-troubleshoot"
+ if verbose:
+ cmdStr += " -vv"
+ if ipv6:
+ cmdStr += " -et ipv6"
+ cmdStr += " -s " + str( sAddr )
+ cmdStr += " -sp " + str( location[ "elementId" ] ) + "/" + str( location[ "port" ] )
+ cmdStr += " -sm " + str( sHost[ "mac" ] )
+ if sHost[ "vlan" ] != "None":
+ cmdStr += " -vid " + sHost[ "vlan" ]
+ cmdStr += " -d " + str( dAddr )
+ netcfg = self.netcfg( args="devices {}".format( location[ "elementId" ] ) )
+ netcfg = json.loads( netcfg )
+ assert netcfg, "Failed to get netcfg"
+ cmdStr += " -dm " + str( netcfg[ "segmentrouting" ][ "routerMac" ] )
+ cmdList.append( cmdStr )
+ return cmdList
except AssertionError:
main.log.exception( "" )
return None
@@ -6570,20 +6576,24 @@
def t3( self, sAddr, dAddr, ipv6=False ):
"""
- Run t3-troubleshoot cli command for given source and destination addresses
+ Run t3-troubleshoot cli commands for all posible routes given source and destination addresses
Options:
sAddr: IP address of the source host
dAddr: IP address of the destination host
"""
try:
- cmdStr = self.composeT3Command( sAddr, dAddr, ipv6 )
- handle = self.sendline( cmdStr )
- assert handle is not None, "Error in sendline"
- assert "Command not found:" not in handle, handle
- assert "Unsupported command:" not in handle, handle
- assert "Error executing command" not in handle, handle
- assert "Tracing packet" in handle
- return handle
+ cmdList = self.composeT3Command( sAddr, dAddr, ipv6 )
+ assert cmdList is not None, "composeT3Command returned None"
+ t3Output = ""
+ for cmdStr in cmdList:
+ handle = self.sendline( cmdStr )
+ assert handle is not None, "Error in sendline"
+ assert "Command not found:" not in handle, handle
+ assert "Unsupported command:" not in handle, handle
+ assert "Error executing command" not in handle, handle
+ assert "Tracing packet" in handle
+ t3Output += handle
+ return t3Output
except AssertionError:
main.log.exception( "" )
return None
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index be64d99..a666d77 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -928,12 +928,14 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
- def onosCli( self, ONOSIp, cmdstr ):
+ def onosCli( self, ONOSIp, cmdstr, timeout=60 ):
"""
Uses 'onos' command to send various ONOS CLI arguments.
Required:
* ONOSIp: specify the ip of the cell machine
* cmdstr: specify the command string to send
+ Optional:
+ * timeout: pexpect timeout for running the command
This function is intended to expose the entire karaf
CLI commands for ONOS. Try to use this function first
@@ -960,15 +962,20 @@
self.handle.expect( self.prompt )
self.handle.sendline( "onos " + ONOSIp + " " + cmdstr )
- self.handle.expect( self.prompt )
-
- handleBefore = self.handle.before
- main.log.info( "Command sent successfully" )
- # Obtain return handle that consists of result from
- # the onos command. The string may need to be
- # configured further.
- returnString = handleBefore
- return returnString
+ i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT ], timeout=timeout )
+ if i == 0:
+ handleBefore = self.handle.before
+ main.log.info( "Command sent successfully" )
+ # Obtain return handle that consists of result from
+ # the onos command. The string may need to be
+ # configured further.
+ returnString = handleBefore
+ return returnString
+ elif i == 1:
+ main.log.error( self.name + ": Timeout when sending " + cmdstr )
+ self.handle.sendline( "\x03" ) # Control-C
+ self.handle.expect( self.prompt )
+ return main.FALSE
except pexpect.TIMEOUT:
main.log.exception( self.name + ": Timeout when sending " + cmdstr )
return main.FALSE
@@ -1614,7 +1621,7 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
- def dumpONOSCmd( self, ONOSIp, CMD, destDir, filename, options="" ):
+ def dumpONOSCmd( self, ONOSIp, CMD, destDir, filename, options="", timeout=60 ):
"""
Dump Cmd to a desired directory.
For debugging purposes, you may want to use
@@ -1627,7 +1634,9 @@
* destDir: specify directory to copy to.
ex ) /tmp/
* fileName: Name of the file
+ Optional:
* options: Options for ONOS command
+ * timeout: pexpect timeout for running the ONOS command
"""
localtime = time.strftime( '%x %X' )
@@ -1637,7 +1646,7 @@
if destDir[ -1: ] != "/":
destDir += "/"
cmd = CMD + " " + options + " > " + str( destDir ) + str( filename ) + localtime
- return self.onosCli( ONOSIp, cmd )
+ return self.onosCli( ONOSIp, cmd, timeout=timeout )
def cpLogsToDir( self, logToCopy,
destDir, copyFileName="" ):
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index c391c22..ed96a23 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -120,13 +120,15 @@
lib.verifyPing( main,
[ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
[ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ],
- stepMsg="Verify reachability from connected internal IPv4 hosts to external IPv4 hosts" )
+ stepMsg="Verify reachability from connected internal IPv4 hosts to external IPv4 hosts",
+ t3Simple=False )
if ipv6:
lib.verifyPing( main,
[ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
[ h for h in main.externalIpv6Hosts if h not in main.disconnectedExternalIpv6Hosts ],
ipv6=True, acceptableFailed=7,
- stepMsg="Verify reachability from connected internal IPv6 hosts to external IPv6 hosts" )
+ stepMsg="Verify reachability from connected internal IPv6 hosts to external IPv6 hosts",
+ t3Simple=False )
# Verify disconnected hosts
if disconnected:
# Disconnected internal to connected external
@@ -134,25 +136,29 @@
lib.verifyPing( main, main.disconnectedIpv4Hosts,
[ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ],
expect=False,
- stepMsg="Verify unreachability of disconnected internal IPv4 hosts to connected external IPv4 hosts" )
+ stepMsg="Verify unreachability of disconnected internal IPv4 hosts to connected external IPv4 hosts",
+ t3Simple=False )
if main.disconnectedIpv6Hosts:
lib.verifyPing( main, main.disconnectedIpv6Hosts,
[ h for h in main.externalIpv6Hosts if h not in main.disconnectedExternalIpv6Hosts ],
ipv6=True, expect=False,
- stepMsg="Verify unreachability of disconnected internal IPv6 hosts to connected external IPv6 hosts" )
+ stepMsg="Verify unreachability of disconnected internal IPv6 hosts to connected external IPv6 hosts",
+ t3Simple=False )
# Connected internal to disconnected external
if main.disconnectedExternalIpv4Hosts:
lib.verifyPing( main,
[ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
main.disconnectedExternalIpv4Hosts,
expect=False,
- stepMsg="Verify unreachability of connected internal IPv4 hosts to disconnected external IPv4 hosts" )
+ stepMsg="Verify unreachability of connected internal IPv4 hosts to disconnected external IPv4 hosts",
+ t3Simple=False )
if main.disconnectedExternalIpv6Hosts:
lib.verifyPing( main,
[ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
main.disconnectedExternalIpv6Hosts,
ipv6=True, expect=False,
- stepMsg="Verify unreachability of connected internal IPv6 hosts to disconnected external IPv6 hosts" )
+ stepMsg="Verify unreachability of connected internal IPv6 hosts to disconnected external IPv6 hosts",
+ t3Simple=False )
def verifyPing( main, ipv4=True, ipv6=True, disconnected=False, internal=True, external=True ):
"""
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index e9bb8fe..ad39667 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -1179,7 +1179,8 @@
@staticmethod
def verifyPing( main, srcList, dstList, ipv6=False, expect=True, wait=1,
- acceptableFailed=0, skipOnFail=True, stepMsg="Verify Ping" ):
+ acceptableFailed=0, skipOnFail=True, stepMsg="Verify Ping",
+ t3Simple=True ):
"""
Verify reachability from each host in srcList to each host in dstList
"""
@@ -1189,7 +1190,7 @@
except ( NameError, AttributeError ):
main.topo = Topology()
main.step( stepMsg )
- pingResult = main.topo.ping( srcList, dstList, ipv6, expect, wait, acceptableFailed, skipOnFail, True )
+ pingResult = main.topo.ping( srcList, dstList, ipv6, expect, wait, acceptableFailed, skipOnFail, t3Simple )
utilities.assert_equals( expect=main.TRUE,
actual=pingResult,
onpass="{}: Pass".format( stepMsg ),
diff --git a/TestON/tests/dependencies/topology.py b/TestON/tests/dependencies/topology.py
index be3b69a..e9fe8d7 100644
--- a/TestON/tests/dependencies/topology.py
+++ b/TestON/tests/dependencies/topology.py
@@ -293,11 +293,16 @@
srcIp = unexpectedPing[ 0 ]
dstIp = unexpectedPing[ 1 ]
main.log.debug( "Collecting t3 with source {} and destination {}".format( srcIp, dstIp ) )
- cmd = main.Cluster.active( 0 ).CLI.composeT3Command( srcIp, dstIp, ipv6, True, t3Simple )
- main.log.debug( "t3 command: {}".format( cmd ) )
- if cmd:
+ cmdList = main.Cluster.active( 0 ).CLI.composeT3Command( srcIp, dstIp, ipv6, True, t3Simple )
+ if not cmdList:
+ main.log.warn( "Failed to compose T3 command with source {} and destination {}".format( srcIp, dstIp ) )
+ continue
+ for i in range( 0, len( cmdList ) ):
+ cmd = cmdList[ i ]
+ main.log.debug( "t3 command: {}".format( cmd ) )
main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, cmd, main.logdir,
- "t3-CASE{}-{}-{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp ) )
+ "t3-CASE{}-{}-{}-route{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp, i ),
+ timeout=10 )
return main.FALSE if unexpectedPings else main.TRUE
def sendScapyPackets( self, sender, receiver, pktFilter, pkt, sIface=None, dIface=None, expect=True, acceptableFailed=0, collectT3=True, t3Command="" ):