Look at port stats to choose port
- In TOST port failure tests, look at port stats to
determine which link is carrying the most traffic
so we can bring it down
- Add eNB leaf-spine link down case
Change-Id: Ia13f3d41e836deaf21dd93574a39ccd954dd488f
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
index cb37498..2d99850 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
@@ -35,7 +35,7 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 20</traffic_cmd_arguments>
+ <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
@@ -63,6 +63,7 @@
<timers>
<LinkDiscovery>12</LinkDiscovery>
<SwitchDiscovery>12</SwitchDiscovery>
+ <TrafficDiscovery>10</TrafficDiscovery>
</timers>
<SLEEP>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
index 124fd50..fd33a86 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
@@ -5,17 +5,48 @@
def CASE1( self, main ):
main.case("Testing connections")
main.persistentSetup = True
- def CASE7( self, main ):
- """
- Tests connectivity between two untagged hosts
- (Ports are configured as vlan-untagged)
- Sets up 3 ONOS instance
- Start 2x2 leaf-spine topology
- Pingall
+ def CASE2( self, main ):
+ """
+ Connect to Pod
+ Perform rolling ONOS failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
+
+ def CASE3( self, main ):
+ """
+ Connect to Pod
+ Perform ONL reboot failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
+
+ def CASE4( self, main ):
+ """
+ Connect to Pod
+ Perform Stratum agent failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
+
+ def CASE5( self, main ):
+ """
+ Connect to Pod
+ Perform Switch Power Cycle failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
+
+ def CASE6( self, main ):
+ """
+ Connect to Pod
+ Perform eNB Leaf-Spine Link, portstate failure/recovery test
+ Collect logs and analyze results
"""
try:
from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
except ImportError:
main.log.error( "SRStagingTest not found. Exiting the test" )
main.cleanAndExit()
@@ -23,16 +54,12 @@
main.funcs
except ( NameError, AttributeError ):
main.funcs = SRStagingTest()
- # Load kubeconfig
- # Setup ssh tunnel
- # connect to ONOS CLI
-
+ descPrefix = "eNB_Leaf_Spine_Portstate"
main.funcs.setupTest( main,
- test_idx=7,
topology='2x2staging',
onosNodes=3,
- description="Developing tests on the staging pod" )
+ description="%s tests on the staging pod" % descPrefix )
srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
srcComponentList = []
for name in srcComponentNames:
@@ -41,26 +68,23 @@
main.downtimeResults = {}
-
# TODO: MOVE TO CONFIG FILE
- device = "device:leaf2"
- port1 = "268"
- port2 = "284"
- port3 = "260"
- port4 = "276"
+ device = "device:leaf1"
+ portsList = [ 176, 180, 184, 188 ]
+ port1 = None
+ port2 = None
+ port3 = None
+ port4 = None
- descPrefix = "Upstream_Leaf_Spine_Portstate"
- # TODO: Move most of this logic into linkDown/linkUp
## First Link Down
shortDesc = descPrefix + "-Failure1"
- longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port1 )
- main.funcs.linkDown( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port1 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Second Link Down
shortDesc = descPrefix + "-Failure2"
- longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port2 )
- main.funcs.linkDown( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port2 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## First Link Up
- # TODO Check these are set correctly
shortDesc = descPrefix + "-Recovery1"
longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port1 )
main.funcs.linkUp( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
@@ -70,12 +94,86 @@
main.funcs.linkUp( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
## Third Link Down
shortDesc = descPrefix + "-Failure3"
- longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port3 )
- main.funcs.linkDown( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port3 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Forth Link Down
shortDesc = descPrefix + "-Failure4"
- longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port4 )
- main.funcs.linkDown( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Third Link Up
+ shortDesc = descPrefix + "-Recovery3"
+ longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Forth Link Up
+ shortDesc = descPrefix + "-Recovery4"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
+
+ main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+ main.funcs.cleanup( main )
+
+ def CASE7( self, main ):
+ """
+ Connect to Pod
+ Perform Upstream Leaf-Spine Link, portstate failure/recovery test
+ Collect logs and analyze results
+ """
+ try:
+ from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
+ except ImportError:
+ main.log.error( "SRStagingTest not found. Exiting the test" )
+ main.cleanAndExit()
+ try:
+ main.funcs
+ except ( NameError, AttributeError ):
+ main.funcs = SRStagingTest()
+
+ descPrefix = "Upstream_Leaf_Spine_Portstate"
+ main.funcs.setupTest( main,
+ topology='2x2staging',
+ onosNodes=3,
+ description="%s tests on the staging pod" % descPrefix )
+ srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
+ srcComponentList = []
+ for name in srcComponentNames:
+ srcComponentList.append( getattr( main, name ) )
+ dstComponent = getattr( main, main.params[ 'PERF' ][ 'pcap_host' ] )
+
+ main.downtimeResults = {}
+
+ # TODO: MOVE TO CONFIG FILE
+ device = "device:leaf2"
+ portsList = [260, 268, 276, 284 ]
+ port1 = None
+ port2 = None
+ port3 = None
+ port4 = None
+
+ ## First Link Down
+ shortDesc = descPrefix + "-Failure1"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port1 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Second Link Down
+ shortDesc = descPrefix + "-Failure2"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port2 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## First Link Up
+ shortDesc = descPrefix + "-Recovery1"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port1 )
+ main.funcs.linkUp( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Second Link Up
+ shortDesc = descPrefix + "-Recovery2"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port2 )
+ main.funcs.linkUp( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Third Link Down
+ shortDesc = descPrefix + "-Failure3"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port3 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Forth Link Down
+ shortDesc = descPrefix + "-Failure4"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
@@ -86,6 +184,5 @@
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
main.log.warn( main.downtimeResults )
- import json
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
index 8c25811..15554bb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
@@ -16,8 +16,8 @@
<diff_clihost>True</diff_clihost> # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
<karaf_username>karaf</karaf_username>
<karaf_password>karaf</karaf_password>
- <web_user>sdn</web_user>
- <web_pass>rocks</web_pass>
+ <web_user>karaf</web_user>
+ <web_pass>karaf</web_pass>
<rest_port></rest_port>
<prompt></prompt> # TODO: we technically need a few of these, one per component
<onos_home>~/Projects/onos/</onos_home> # defines where onos home is on the target cell machine. Defaults to entry in "home" if empty.
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index 5dd43da..50f3190 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -45,7 +45,7 @@
self.switchNames[ '2x2' ] = [ "leaf1", "leaf2", "spine101", "spine102" ]
main.switchType = "ovs"
- def setupTest( self, main, test_idx, topology, onosNodes, description, vlan = [] ):
+ def setupTest( self, main, topology, onosNodes, description, vlan = [] ):
try:
skipPackage = False
init = False
@@ -62,7 +62,6 @@
onosNodes,
's' if onosNodes > 1 else '' ) )
- main.cfgName = 'CASE%01d%01d' % ( test_idx / 10, ( ( test_idx - 1 ) % 10 ) % 4 + 1 )
main.Cluster.setRunningNode( onosNodes )
# Set ONOS Log levels
# TODO: Check levels before and reset them after
@@ -224,25 +223,56 @@
except Exception as e:
main.log.exception( "Error in stopCapturing" )
- def linkDown( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc ):
+ def linkDown( self, device, portsList, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
""""
High level function that handles an event including monitoring
Arguments:
device - String of the device uri in ONOS
- port - String of the port uri in ONOS
+ portsList - List of strings of the port uri in ONOS that we might take down
srcComponentLsit - List containing src components, used for sending traffic
dstComponent - Component used for receiving taffic
shortDesc - String, Short description, used in reporting and file prefixes
longDesc - String, Longer description, used in logging
+ Returns:
+ A string of the port id that was brought down
"""
import time
+ deltaStats = {}
+ for p in portsList:
+ deltaStats[ p ] = {}
try:
+ # Get port stats info
+ initialStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ for d in initialStats:
+ if d[ 'device' ] == device:
+ for p in d[ 'ports' ]:
+ if p[ 'port' ] in portsList:
+ deltaStats[ p[ 'port' ] ][ 'tx1' ] = p[ 'packetsSent' ]
+
main.step( "Start Capturing" )
main.funcs.startCapturing( main,
srcComponentList,
dstComponent,
shortDesc=shortDesc,
longDesc=longDesc )
+ # Let some packets flow
+ time.sleep( float( main.params['timers'].get( 'TrafficDiscovery', 5 ) ) )
+ # Get port stats info
+ updatedStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ for d in updatedStats:
+ if d[ 'device' ] == device:
+ for p in d[ 'ports' ]:
+ if p[ 'port' ] in portsList:
+ deltaStats[ p[ 'port' ] ][ 'tx2' ] = p[ 'packetsSent' ]
+ for port, stats in deltaStats.iteritems():
+ deltaStats[ port ]['delta'] = stats[ 'tx2' ] - stats[ 'tx1' ]
+
+ main.log.debug( deltaStats )
+ port = max( deltaStats, key=lambda p: deltaStats[ p ][ 'tx2' ] - deltaStats[ p ][ 'tx1' ] )
+ if deltaStats[ port ][ 'delta' ] == 0:
+ main.log.warn( "Could not find a port with traffic. Likely need to wait longer for stats to be updated" )
+ main.log.debug( port )
+ # Determine which port to bring down
main.step( "Port down" )
ctrl = main.Cluster.active( 0 ).CLI
portDown = ctrl.portstate( dpid=device, port=port, state="disable" )
@@ -254,18 +284,19 @@
adminState = p['isEnabled']
main.log.debug( adminState )
#TODO ASSERTS
- main.log.info( "Sleeping 10 seconds" )
- time.sleep(10)
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
main.step( "Stop Capturing" )
main.funcs.stopCapturing( main,
srcComponentList,
dstComponent,
shortDesc=shortDesc,
longDesc=longDesc )
+ return port
except Exception as e:
main.log.exception( "Error in linkDown" )
- def linkUp( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc ):
+ def linkUp( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
""""
High level function that handles an event including monitoring
Arguments:
@@ -277,6 +308,9 @@
longDesc - String, Longer description, used in logging
"""
import time
+ if port is None:
+ main.log.warn( "Incorrect port number, cannot bring up port" )
+ return
try:
main.step( "Start Capturing" )
main.funcs.startCapturing( main,
@@ -295,8 +329,8 @@
adminState = p['isEnabled']
main.log.debug( adminState )
#TODO ASSERTS
- main.log.info( "Sleeping 10 seconds" )
- time.sleep(10)
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
main.step( "Stop Capturing" )
main.funcs.stopCapturing( main,
srcComponentList,
@@ -387,7 +421,6 @@
main.log.warn( "Error opening " + dbFileName + " to write results." )
def cleanup( self, main ):
- # TODO: Do things like restore log levels here
run.cleanup( main )
+ main.step( "Writing csv results file for db" )
self.dbWrite( main, "SRStaging-dbfile.csv")
-