Merge "Handling errors during nightly runs"
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 7d306b9..0dc3787 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -57,7 +57,7 @@
ssh_newkey = 'Are you sure you want to continue connecting'
refused = "ssh: connect to host " + \
self.ip_address + " port 22: Connection refused"
- ssh_options = "-t -X -A -o ServerAliveInterval=120 -o TCPKeepAlive=yes"
+ ssh_options = "-t -X -A -o ServerAliveInterval=50 -o TCPKeepAlive=yes"
ssh_destination = self.user_name + "@" + self.ip_address
envVars = { "TERM": "vt100" }
# TODO: Add option to specify which shell/command to use
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
index 668a7a6..a292173 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
@@ -639,8 +639,14 @@
fabricIntfIp = ipMatch.group(1)
main.log.debug( "Found %s as gateway ip for %s" % ( fabricIntfIp, srcComponent.shortName ) )
# FIXME: How to chose the correct one if there are multiple? look at subnets
- srcComponent.addRouteToHost( route, fabricIntfIp, srcIface, sudoRequired=True, purgeOnDisconnect=True )
+ addResult = srcComponent.addRouteToHost( route, fabricIntfIp, srcIface, sudoRequired=True, purgeOnDisconnect=True )
+ failMsg = "Failed to add static route to host"
+ utilities.assert_equals( expect=main.TRUE, actual=addResult,
+ onpass="Added static route to host",
+ onfail=failMsg )
main.log.debug( srcComponent.getRoutes() )
+ if not addResult:
+ main.skipCase( result="FAIL", msg=failMsg )
# Add route in ONOS
nextHopIface = nextHopComponent.interfaces[0].get( 'name' )
@@ -667,7 +673,10 @@
main.Cluster.active( 0 ).routeRemove( route, nextHopIp )
main.log.debug( main.Cluster.active( 0 ).routes() )
# Remove route on host
- srcComponent.deleteRoute( route, fabricIntfIp, srcIface, sudoRequired=True )
+ delResult = srcComponent.deleteRoute( route, fabricIntfIp, srcIface, sudoRequired=True )
+ utilities.assert_equals( expect=main.TRUE, actual=delResult,
+ onpass="Removed static route from host",
+ onfail="Failed to remove static route from host" )
main.log.debug( srcComponent.getRoutes() )
# Cleanup
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index f0f30a5..4213e0d 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -148,16 +148,23 @@
src.interfaces[0]['name'] ) )
try:
i = src.handle.expect( [ "password", src.prompt ] )
+ output = src.handle.before + src.handle.after
if i == 0:
src.handle.sendline( src.pwd )
i = src.handle.expect( [ "password", src.prompt ] )
assert i != 0, "Incorrect Password"
+ output = src.handle.before + src.handle.after
except Exception:
main.log.exception( "%s: Unexpected response from ping" % src.name )
src.handle.send( '\x03' ) # ctrl-c
src.handle.expect( src.prompt )
+ return main.FALSE
main.funcs.clearBuffer( src )
main.log.warn( "%s: %s" % ( src.name, str( src.handle.before ) ) )
+ if " 0% packet loss" in output:
+ return main.TRUE
+ else:
+ return main.FALSE
@staticmethod
def startIperf( main, src, dstIp, trafficSelector, trafficDuration ):
@@ -178,6 +185,8 @@
trafficDuration )
main.log.info( "Starting iperf between %s and %s" % ( src.shortName, dstIp ) )
sudoCheck = main.funcs.singlePingWithSudo( main, src, dstIp )
+ if not sudoCheck:
+ main.skipCase( result="FAIL", msg="Incorrect permissions for sudo" )
src.handle.sendline( "/usr/bin/iperf %s " % iperfArgs )
src.preDisconnect = src.exitFromProcess
@@ -249,8 +258,20 @@
dstIp = dst.interfaces[0]['ips'][0]
main.log.info( "Setting up flow between %s and %s%s" % ( src.shortName, dst.shortName, "" if not dstIp else " with dstIp %s" % dstIp ) )
# ping right before to make sure arp is cached and sudo is authenticated
- main.funcs.singlePingWithSudo( main, src, dst.interfaces[0]['ips'][0] )
- main.funcs.singlePingWithSudo( main, dst, src.interfaces[0]['ips'][0] )
+ sudoCheck1 = main.funcs.singlePingWithSudo( main, src, dst.interfaces[0]['ips'][0] )
+ checkDesc = "sudo ping from %s to %s" % ( src.shortName, dst.interfaces[0]['ips'][0] )
+ if not sudoCheck1:
+ main.skipCase( result="FAIL", msg="Incorrect permissions for %s" % checkDesc )
+ utilities.assert_equals( expect=main.TRUE, actual=sudoCheck1,
+ onpass="Successfully %s" % checkDesc ,
+ onfail="Failed to %s" % checkDesc )
+ sudoCheck2 = main.funcs.singlePingWithSudo( main, dst, src.interfaces[0]['ips'][0] )
+ checkDesc = "sudo ping from %s to %s" % ( dst.shortName, src.interfaces[0]['ips'][0] )
+ if not sudoCheck2:
+ main.skipCase( result="FAIL", msg="Incorrect permissions for %s" % checkDesc )
+ utilities.assert_equals( expect=main.TRUE, actual=sudoCheck1,
+ onpass="Successfully %s" % checkDesc ,
+ onfail="Failed to %s" % checkDesc )
# Start traffic
# TODO: ASSERTS
if pingOnly: