Enable running tests on a persistant network setup
- Doesn't setup or teardown the control or data planes
- Increase timeout for reading scapy packets
TODO:
- Add to all params files for qa pod
- Update tofino switch driver
- Test on other test suites
Change-Id: I8de8e10205b508fd7cd7fb49fc7715369dab5fb0
(cherry picked from commit bb1e6baec7126ca99240bab1e45622eec037021b)
diff --git a/TestON/drivers/common/cli/emulator/scapyclidriver.py b/TestON/drivers/common/cli/emulator/scapyclidriver.py
index bcaebfb..d340d5c 100644
--- a/TestON/drivers/common/cli/emulator/scapyclidriver.py
+++ b/TestON/drivers/common/cli/emulator/scapyclidriver.py
@@ -727,7 +727,7 @@
# clear buffer
if debug:
main.log.warn( "%s expect loop iteration" % i )
- self.handle.expect( self.scapyPrompt, timeout=1 )
+ self.handle.expect( self.scapyPrompt, timeout=5 )
response += self.cleanOutput( self.handle.before, debug )
except pexpect.TIMEOUT:
return response
diff --git a/TestON/drivers/common/cli/networkdriver.py b/TestON/drivers/common/cli/networkdriver.py
index c151aab..0f7b699 100755
--- a/TestON/drivers/common/cli/networkdriver.py
+++ b/TestON/drivers/common/cli/networkdriver.py
@@ -532,12 +532,12 @@
# Use scapy to send and recieve packets
hostPair[ 1 ].startScapy( ifaceName=dstIface )
hostPair[ 1 ].addRoutes()
- hostPair[ 1 ].startFilter( ifaceName=dstIface, pktFilter="ether host %s and ip host %s" % ( srcMac, srcIPs[0] ) )
+ hostPair[ 1 ].startFilter( ifaceName=dstIface, pktFilter="ether src host %s and ip src host %s" % ( srcMac, srcIPs[0] ) )
hostPair[ 0 ].startScapy( ifaceName=srcIface )
hostPair[ 0 ].addRoutes()
- hostPair[ 0 ].buildEther( dst=dstMac )
- hostPair[ 0 ].buildIP( src=srcIPs, dst=dstIPs )
+ hostPair[ 0 ].buildEther( src=srcMac, dst=dstMac )
+ hostPair[ 0 ].buildIP( src=srcIPs[0], dst=dstIPs[0] )
hostPair[ 0 ].buildICMP( )
hostPair[ 0 ].sendPacket( iface=srcIface )
@@ -545,7 +545,7 @@
if not waiting:
pingResult = main.FALSE
packets = hostPair[ 1 ].readPackets()
- main.log.warn( packets )
+ main.log.warn( repr( packets ) )
for packet in packets.splitlines():
main.log.debug( packet )
if srcIPs[0] in packet:
@@ -625,7 +625,7 @@
main.log.debug( "Pinging from " + str( srcHost.shortName ) + " to " + str( dstHost.shortName ) )
# Use scapy to send and recieve packets
dstHost.startFilter()
- srcHost.buildEther( dst=srcHost.interfaces[0].get( 'mac') )
+ srcHost.buildEther( src=srcHost.interfaces[0].get( 'mac'), dst=dstHost.interfaces[0].get( 'mac') )
srcHost.sendPacket()
output = dstHost.checkFilter()
main.log.debug( output )
@@ -781,11 +781,12 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
- def getIPAddress( self, host, proto='IPV4' ):
+ def getIPAddress( self, host, iface=None, proto='IPV4' ):
"""
Returns IP address of the host
"""
- response = self.runCmdOnHost( host, "ifconfig" )
+ cmd = "ifconfig %s" % iface if iface else ""
+ response = self.runCmdOnHost( host, cmd )
pattern = ''
if proto == 'IPV4':
pattern = "inet\s(\d+\.\d+\.\d+\.\d+)\s\snetmask"
@@ -1049,11 +1050,13 @@
for host in hostList:
flushCmd = ""
cmd = ""
- if self.getIPAddress( host ) or hosts[host]['interfaces'][0].get( 'ips', False ) :
+ intf = hosts[host]['interfaces'][0].get( 'name' )
+ hostIp = self.getIPAddress( host, iface=intf ) or hosts[host]['interfaces'][0].get( 'ips', False )
+ if hostIp:
flushCmd = "sudo ip neigh flush all"
- intf = hosts[host]['interfaces'][0].get( 'name' )
intfStr = "-i {}".format( intf ) if intf else ""
- cmd = "sudo arping -c 1 -w {} {} {}".format( wait, intfStr, dstIp )
+ srcIp = "-S {}".format( hostIp if isinstance( hostIp, types.StringType ) else hostIp[0] ) if intf else ""
+ cmd = "sudo arping -c 1 -w {} {} {} {}".format( wait, intfStr, srcIp, dstIp )
main.log.debug( "Sending IPv4 arping from host {}".format( host ) )
elif self.getIPAddress( host, proto='IPV6' ):
flushCmd = "sudo ip -6 neigh flush all"
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index ef69aec..b0bf7cb 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -3318,7 +3318,7 @@
result = main.TRUE
else:
output = output + \
- "The number of links, switches, nodes, and SCCs does not match " + \
+ "The number of links, switches, nodes, and SCCs does not match " + \
"what was expected"
result = main.FALSE
output = output + "\n ONOS sees %i devices " % int( devices )
@@ -6774,7 +6774,7 @@
output = ""
for cmdStr in cmdList:
self.handle.sendline( cmdStr )
- self.handle.expect( self.dockerPrompt, timeout=120 )
+ self.handle.expect( self.dockerPrompt, timeout=420 )
self.handle.sendline( "" )
self.handle.expect( self.dockerPrompt )
handle = self.handle.before
diff --git a/TestON/drivers/common/cli/stratumosswitchdriver.py b/TestON/drivers/common/cli/stratumosswitchdriver.py
index 3eae155..1490079 100644
--- a/TestON/drivers/common/cli/stratumosswitchdriver.py
+++ b/TestON/drivers/common/cli/stratumosswitchdriver.py
@@ -106,8 +106,9 @@
try:
if self.handle:
# Stop the agent
- self.stopSwitchAgent()
- main.log.debug( self.name + ": Disconnected" )
+ if not main.persistentSetup:
+ self.stopSwitchAgent()
+ main.log.debug( self.name + ": Disconnected" )
# Disconnect from the device
self.handle.sendline( "" )
self.handle.expect( self.prompt )
diff --git a/TestON/tests/HA/dependencies/HA.py b/TestON/tests/HA/dependencies/HA.py
index adc4d5f..9af5dfb 100644
--- a/TestON/tests/HA/dependencies/HA.py
+++ b/TestON/tests/HA/dependencies/HA.py
@@ -3526,7 +3526,7 @@
description = "Check that Leadership Election is still functional"
main.case( description )
- # NOTE: Need to re-run after restarts since being a canidate is not persistant
+ # NOTE: Need to re-run after restarts since being a candidate is not persistent
oldLeaders = [] # list of lists of each nodes' candidates before
newLeaders = [] # list of lists of each nodes' candidates after
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.tofino b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.tofino
index fc89308..7442bf4 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.tofino
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.tofino
@@ -25,6 +25,8 @@
<jsonFileSuffix>.hw</jsonFileSuffix>
+ <persistent_setup>True</persistent_setup>
+
<MN_DOCKER>
<args>--privileged --net host --rm -v topo:/topo -v ~/mininet/custom:/home/root/mininet/custom -v /var/run/openvswitch/:/var/run/openvswitch/ -v /tmp/mn-stratum:/tmp -v /tmp/mn_conf/:/home/root/config --hostname mn-stratum -v /etc/network/interfaces:/etc/network/interfaces -it -d</args>
<name>trellis_mininet</name>
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 15c09d5..699370c 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -49,6 +49,7 @@
main.log.error( "ONOSSetup not found. exiting the test" )
main.cleanAndExit()
from tests.dependencies.Network import Network
+ main.persistentSetup = main.params.get( "persistent_setup" )
main.Network = Network()
main.physicalNet = False
main.testSetUp.envSetupDescription( False )
@@ -112,7 +113,7 @@
"""
# main.scale[ 0 ] determines the current number of ONOS controller
try:
- if main.params.get( 'EXTERNAL_APPS' ):
+ if not main.persistentSetup and main.params.get( 'EXTERNAL_APPS' ):
for app, url in main.params[ 'EXTERNAL_APPS' ].iteritems():
main.log.info( "Downloading %s app from %s" % ( app, url ) )
main.ONOSbench.onosFetchApp( url )
@@ -170,6 +171,19 @@
except Exception as e:
main.log.error( e )
+ # Install segmentrouting and t3 app
+ appInstallResult = main.TRUE
+ if not main.persistentSetup:
+ if main.trellisOar:
+ appInstallResult = appInstallResult and main.ONOSbench.onosAppInstall( main.Cluster.runningNodes[0].ipAddress, main.trellisOar)
+ if main.t3Oar:
+ appInstallResult = appInstallResult and main.ONOSbench.onosAppInstall( main.Cluster.runningNodes[0].ipAddress, main.t3Oar)
+ utilities.assert_equals( expect=main.TRUE, actual=appInstallResult,
+ onpass="SR app installation succeded",
+ onfail="SR app installation failed" )
+ if not appInstallResult:
+ main.cleanAndExit()
+
Testcaselib.setOnosLogLevels( main )
Testcaselib.setOnosConfig( main )
@@ -364,31 +378,32 @@
if not topoResult:
main.cleanAndExit()
- # Perform any optional setup
- for switch in main.NetworkBench.switches:
- if hasattr( switch, "setup" ):
- switch.setup() # We might not need this
+ if not main.persistentSetup:
+ # Perform any optional setup
+ for switch in main.NetworkBench.switches:
+ if hasattr( switch, "setup" ):
+ switch.setup() # We might not need this
- main.step( "Assign switches to controllers." )
- stepResult = main.TRUE
- switches = main.NetworkBench.getSwitches()
- pool = []
- for name in switches.keys():
- # NOTE: although this terminology is ovsdb centric, we can use this function for other switches too
- # e.g. push onos net-cfg for stratum switches
- thread = main.Thread( target=main.NetworkBench.assignSwController,
- name="assignSwitchToController",
- args=[ name, main.Cluster.getIps(), '6653' ] )
- pool.append( thread )
- thread.start()
- for thread in pool:
- thread.join( 300 )
- if not thread.result:
- stepResult = main.FALSE
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="Successfully assign switches to controllers",
- onfail="Failed to assign switches to controllers" )
+ main.step( "Assign switches to controllers." )
+ stepResult = main.TRUE
+ switches = main.NetworkBench.getSwitches()
+ pool = []
+ for name in switches.keys():
+ # NOTE: although this terminology is ovsdb centric, we can use this function for other switches too
+ # e.g. push onos net-cfg for stratum switches
+ thread = main.Thread( target=main.NetworkBench.assignSwController,
+ name="assignSwitchToController",
+ args=[ name, main.Cluster.getIps(), '6653' ] )
+ pool.append( thread )
+ thread.start()
+ for thread in pool:
+ thread.join( 300 )
+ if not thread.result:
+ stepResult = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully assign switches to controllers",
+ onfail="Failed to assign switches to controllers" )
# Check devices
Testcaselib.checkDevices( main, switches=int( main.params[ 'TOPO' ][ 'switchNum' ] ) )
@@ -1000,8 +1015,9 @@
"""
from tests.dependencies.utils import Utils
main.utils = Utils()
- for ctrl in main.Cluster.active():
- ctrl.CLI.log( "\"Ending Test - Shutting down ONOS and Network\"", level="INFO" )
+ if not main.persistentSetup:
+ for ctrl in main.Cluster.active():
+ ctrl.CLI.log( "\"Ending Test - Shutting down ONOS and Network\"", level="INFO" )
# Clean up scapy hosts
if hasattr( main, "scapyHosts" ):
scapyResult = main.TRUE
@@ -1034,8 +1050,9 @@
if copyKarafLog:
main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True, includeCaseDesc=False )
- for ctrl in main.Cluster.active():
- main.ONOSbench.onosStop( ctrl.ipAddress )
+ if not main.persistentSetup:
+ for ctrl in main.Cluster.active():
+ main.ONOSbench.onosStop( ctrl.ipAddress )
Testcaselib.mnDockerTeardown( main )
@staticmethod
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
index 5808ba2..aa63714 100644
--- a/TestON/tests/dependencies/Cluster.py
+++ b/TestON/tests/dependencies/Cluster.py
@@ -752,7 +752,7 @@
if ips == activeIps:
currentResult = True
else:
- main.log.warn( "{} != {}".format( ips, activeIps ) )
+ main.log.warn( "Expected {} != found {}".format( ips, activeIps ) )
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index f904730..dc1dea0 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -26,6 +26,10 @@
def __init__( self ):
self.default = ''
+ try:
+ main.persistentSetup
+ except ( NameError, AttributeError ):
+ main.persistentSetup = False
def envSetupDescription( self, includeCaseDesc=True ):
"""
@@ -605,11 +609,12 @@
Returns main.TRUE if it everything successfully proceeded.
"""
self.setNumCtrls( hasMultiNodeRounds )
- if includeCaseDesc:
- main.case( "Starting up " + str( cluster.numCtrls ) +
- " node(s) ONOS cluster" )
- main.caseExplanation = "Set up ONOS with " + str( cluster.numCtrls ) + \
- " node(s) ONOS cluster"
+ if not main.persistentSetup:
+ if includeCaseDesc:
+ main.case( "Starting up " + str( cluster.numCtrls ) +
+ " node(s) ONOS cluster" )
+ main.caseExplanation = "Set up ONOS with " + str( cluster.numCtrls ) + \
+ " node(s) ONOS cluster"
main.log.info( "ONOS cluster size = " + str( cluster.numCtrls ) )
cellResult = main.TRUE
@@ -634,7 +639,7 @@
tempOnosIp, installMax,
atomixClusterSize )
- if restartCluster:
+ if not main.persistentSetup and restartCluster:
atomixKillResult = self.killingAllAtomix( cluster, killRemoveMax, stopAtomix )
onosKillResult = self.killingAllOnos( cluster, killRemoveMax, stopOnos )
dockerKillResult = self.killingAllOnosDocker( cluster, killRemoveMax )
@@ -642,7 +647,7 @@
else:
killResult = main.TRUE
- if restartCluster:
+ if not main.persistentSetup and restartCluster:
atomixUninstallResult = self.uninstallAtomix( cluster, killRemoveMax )
onosUninstallResult = self.uninstallOnos( cluster, killRemoveMax )
uninstallResult = atomixUninstallResult and onosUninstallResult
@@ -681,6 +686,9 @@
onosServiceResult = main.TRUE
if not cluster.useDocker:
onosServiceResult = self.checkOnosService( cluster )
+ elif main.persistentSetup:
+ for ctrl in cluster.getRunningNodes():
+ ctrl.inDocker = True
onosCliResult = main.TRUE
if startOnosCli:
@@ -689,7 +697,7 @@
onosNodesResult = self.checkOnosNodes( cluster )
externalAppsResult = main.TRUE
- if main.params.get( 'EXTERNAL_APPS' ):
+ if not main.persistentSetup and main.params.get( 'EXTERNAL_APPS' ):
node = main.Cluster.controllers[0]
for app, url in main.params[ 'EXTERNAL_APPS' ].iteritems():
path, fileName = os.path.split( url )
@@ -697,7 +705,7 @@
onosAppsResult = main.TRUE
- if cellApply:
+ if not main.persistentSetup and cellApply:
if apps:
newApps = []
appNames = apps.split( ',' )