Look at port stats to choose port
- In TOST port failure tests, look at port stats to
determine which link is carrying the most traffic
so we can bring it down
- Add eNB leaf-spine link down case
Change-Id: Ia13f3d41e836deaf21dd93574a39ccd954dd488f
diff --git a/TestON/drivers/common/api/controller/onosrestdriver.py b/TestON/drivers/common/api/controller/onosrestdriver.py
index 7f51bcd..403b436 100755
--- a/TestON/drivers/common/api/controller/onosrestdriver.py
+++ b/TestON/drivers/common/api/controller/onosrestdriver.py
@@ -2345,3 +2345,41 @@
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+
+ def portstats( self, ip="DEFAULT", port="DEFAULT" ):
+ """
+ Description:
+ Gets the portstats for each port in ONOS
+ Returns:
+ A list of dicts containing device id and a list of dicts containing the
+ port statistics for each port.
+ Returns main.FALSE if error on request;
+ Returns None for exception
+ """
+ try:
+ output = None
+ if ip == "DEFAULT":
+ main.log.warn( self.name + ": No ip given, reverting to ip from topo file" )
+ ip = self.ip_address
+ if port == "DEFAULT":
+ main.log.warn( self.name + ": No port given, reverting to port " +
+ "from topo file" )
+ port = self.port
+ response = self.send( url="/statistics/ports", ip = ip, port = port )
+ if response:
+ if 200 <= response[ 0 ] <= 299:
+ output = response[ 1 ]
+ a = json.loads( output ).get( 'statistics' )
+ assert a is not None, "Error parsing json object"
+ b = json.dumps( a )
+ return b
+ else:
+ main.log.error( "Error with REST request, response was: %s: %s" %
+ ( response[ 0 ], response[ 1 ] ) )
+ return main.FALSE
+ except ( AttributeError, AssertionError, TypeError ):
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
diff --git a/TestON/drivers/common/cli/emulator/scapyclidriver.py b/TestON/drivers/common/cli/emulator/scapyclidriver.py
index 79a1956..c4708bc 100644
--- a/TestON/drivers/common/cli/emulator/scapyclidriver.py
+++ b/TestON/drivers/common/cli/emulator/scapyclidriver.py
@@ -792,7 +792,7 @@
self.handle.expect( self.scapyPrompt )
response = self.cleanOutput( self.handle.before )
main.log.debug( self.name + ": Send packet response: {}".format( response ) )
- if "Traceback" in response:
+ if "Traceback" in response or "Errno" in response or "Error" in response:
# KeyError, SyntaxError, ...
main.log.error( self.name + ": Error in sending command: " + response )
return main.FALSE
@@ -915,6 +915,10 @@
else:
self.handle.sendline( "pkts.summary()")
output = self.clearBuffer()
+ if "Traceback" in output or "Errno" in output or "Error" in output:
+ # KeyError, SyntaxError, IOError, NameError, ...
+ main.log.error( self.name + ": Error in sending command: " + output )
+ main.cleanAndExit()
except pexpect.TIMEOUT:
main.log.exception( self.name + ": Command timed out" )
return None
@@ -1081,7 +1085,7 @@
gw = route.get( 'gw' )
iface = route.get( 'interface' )
returnValues .append( self.addRoute( "%s/%s" % ( route.get( 'network' ), route.get( 'netmask' ) ),
- gw if gw else main.Cluster.active(0).ipAddress,
+ gw if gw else main.Cluster.active(0).address,
interface=iface if iface else self.interfaces[ 0 ].get( 'name' ) ) )
return returnValues
diff --git a/TestON/drivers/common/cli/onosclusterdriver.py b/TestON/drivers/common/cli/onosclusterdriver.py
index d10b17f..b0f924b 100755
--- a/TestON/drivers/common/cli/onosclusterdriver.py
+++ b/TestON/drivers/common/cli/onosclusterdriver.py
@@ -233,6 +233,8 @@
portsList += "%s:%s " % ( localPort, port )
if port == cliPort:
node.CLI.karafPort = localPort
+ elif port == guiPort:
+ node.REST.port = localPort
main.log.info( "Setting up port forward for pod %s: [ %s ]" % ( self.podNames[ index ], portsList ) )
pf = kubectl.kubectlPortForward( self.podNames[ index ],
portsList,
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index aa1e9f9..e2265ce 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -1065,7 +1065,7 @@
"""
try:
- cmdStr = "kubectl %s %s get pods -o=custom-columns=NAME:.metadata.name,NODE:.spec.nodeName %s " % (
+ cmdStr = "kubectl %s %s get pods -o wide %s " % (
"--kubeconfig %s" % kubeconfig if kubeconfig else "",
"-n %s" % namespace if namespace else "",
" > %s" % dstPath if dstPath else "" )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
index cb37498..2d99850 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
@@ -35,7 +35,7 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 20</traffic_cmd_arguments>
+ <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
@@ -63,6 +63,7 @@
<timers>
<LinkDiscovery>12</LinkDiscovery>
<SwitchDiscovery>12</SwitchDiscovery>
+ <TrafficDiscovery>10</TrafficDiscovery>
</timers>
<SLEEP>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
index 124fd50..fd33a86 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
@@ -5,17 +5,48 @@
def CASE1( self, main ):
main.case("Testing connections")
main.persistentSetup = True
- def CASE7( self, main ):
- """
- Tests connectivity between two untagged hosts
- (Ports are configured as vlan-untagged)
- Sets up 3 ONOS instance
- Start 2x2 leaf-spine topology
- Pingall
+ def CASE2( self, main ):
+ """
+ Connect to Pod
+ Perform rolling ONOS failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
+
+ def CASE3( self, main ):
+ """
+ Connect to Pod
+ Perform ONL reboot failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
+
+ def CASE4( self, main ):
+ """
+ Connect to Pod
+ Perform Stratum agent failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
+
+ def CASE5( self, main ):
+ """
+ Connect to Pod
+ Perform Switch Power Cycle failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
+
+ def CASE6( self, main ):
+ """
+ Connect to Pod
+ Perform eNB Leaf-Spine Link, portstate failure/recovery test
+ Collect logs and analyze results
"""
try:
from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
except ImportError:
main.log.error( "SRStagingTest not found. Exiting the test" )
main.cleanAndExit()
@@ -23,16 +54,12 @@
main.funcs
except ( NameError, AttributeError ):
main.funcs = SRStagingTest()
- # Load kubeconfig
- # Setup ssh tunnel
- # connect to ONOS CLI
-
+ descPrefix = "eNB_Leaf_Spine_Portstate"
main.funcs.setupTest( main,
- test_idx=7,
topology='2x2staging',
onosNodes=3,
- description="Developing tests on the staging pod" )
+ description="%s tests on the staging pod" % descPrefix )
srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
srcComponentList = []
for name in srcComponentNames:
@@ -41,26 +68,23 @@
main.downtimeResults = {}
-
# TODO: MOVE TO CONFIG FILE
- device = "device:leaf2"
- port1 = "268"
- port2 = "284"
- port3 = "260"
- port4 = "276"
+ device = "device:leaf1"
+ portsList = [ 176, 180, 184, 188 ]
+ port1 = None
+ port2 = None
+ port3 = None
+ port4 = None
- descPrefix = "Upstream_Leaf_Spine_Portstate"
- # TODO: Move most of this logic into linkDown/linkUp
## First Link Down
shortDesc = descPrefix + "-Failure1"
- longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port1 )
- main.funcs.linkDown( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port1 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Second Link Down
shortDesc = descPrefix + "-Failure2"
- longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port2 )
- main.funcs.linkDown( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port2 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## First Link Up
- # TODO Check these are set correctly
shortDesc = descPrefix + "-Recovery1"
longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port1 )
main.funcs.linkUp( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
@@ -70,12 +94,86 @@
main.funcs.linkUp( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
## Third Link Down
shortDesc = descPrefix + "-Failure3"
- longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port3 )
- main.funcs.linkDown( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port3 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Forth Link Down
shortDesc = descPrefix + "-Failure4"
- longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port4 )
- main.funcs.linkDown( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Third Link Up
+ shortDesc = descPrefix + "-Recovery3"
+ longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Forth Link Up
+ shortDesc = descPrefix + "-Recovery4"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
+
+ main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+ main.funcs.cleanup( main )
+
+ def CASE7( self, main ):
+ """
+ Connect to Pod
+ Perform Upstream Leaf-Spine Link, portstate failure/recovery test
+ Collect logs and analyze results
+ """
+ try:
+ from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
+ except ImportError:
+ main.log.error( "SRStagingTest not found. Exiting the test" )
+ main.cleanAndExit()
+ try:
+ main.funcs
+ except ( NameError, AttributeError ):
+ main.funcs = SRStagingTest()
+
+ descPrefix = "Upstream_Leaf_Spine_Portstate"
+ main.funcs.setupTest( main,
+ topology='2x2staging',
+ onosNodes=3,
+ description="%s tests on the staging pod" % descPrefix )
+ srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
+ srcComponentList = []
+ for name in srcComponentNames:
+ srcComponentList.append( getattr( main, name ) )
+ dstComponent = getattr( main, main.params[ 'PERF' ][ 'pcap_host' ] )
+
+ main.downtimeResults = {}
+
+ # TODO: MOVE TO CONFIG FILE
+ device = "device:leaf2"
+ portsList = [260, 268, 276, 284 ]
+ port1 = None
+ port2 = None
+ port3 = None
+ port4 = None
+
+ ## First Link Down
+ shortDesc = descPrefix + "-Failure1"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port1 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Second Link Down
+ shortDesc = descPrefix + "-Failure2"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port2 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## First Link Up
+ shortDesc = descPrefix + "-Recovery1"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port1 )
+ main.funcs.linkUp( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Second Link Up
+ shortDesc = descPrefix + "-Recovery2"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port2 )
+ main.funcs.linkUp( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Third Link Down
+ shortDesc = descPrefix + "-Failure3"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port3 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Forth Link Down
+ shortDesc = descPrefix + "-Failure4"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
@@ -86,6 +184,5 @@
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
main.log.warn( main.downtimeResults )
- import json
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
index 8c25811..15554bb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
@@ -16,8 +16,8 @@
<diff_clihost>True</diff_clihost> # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
<karaf_username>karaf</karaf_username>
<karaf_password>karaf</karaf_password>
- <web_user>sdn</web_user>
- <web_pass>rocks</web_pass>
+ <web_user>karaf</web_user>
+ <web_pass>karaf</web_pass>
<rest_port></rest_port>
<prompt></prompt> # TODO: we technically need a few of these, one per component
<onos_home>~/Projects/onos/</onos_home> # defines where onos home is on the target cell machine. Defaults to entry in "home" if empty.
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index 5dd43da..50f3190 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -45,7 +45,7 @@
self.switchNames[ '2x2' ] = [ "leaf1", "leaf2", "spine101", "spine102" ]
main.switchType = "ovs"
- def setupTest( self, main, test_idx, topology, onosNodes, description, vlan = [] ):
+ def setupTest( self, main, topology, onosNodes, description, vlan = [] ):
try:
skipPackage = False
init = False
@@ -62,7 +62,6 @@
onosNodes,
's' if onosNodes > 1 else '' ) )
- main.cfgName = 'CASE%01d%01d' % ( test_idx / 10, ( ( test_idx - 1 ) % 10 ) % 4 + 1 )
main.Cluster.setRunningNode( onosNodes )
# Set ONOS Log levels
# TODO: Check levels before and reset them after
@@ -224,25 +223,56 @@
except Exception as e:
main.log.exception( "Error in stopCapturing" )
- def linkDown( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc ):
+ def linkDown( self, device, portsList, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
""""
High level function that handles an event including monitoring
Arguments:
device - String of the device uri in ONOS
- port - String of the port uri in ONOS
+ portsList - List of strings of the port uri in ONOS that we might take down
srcComponentLsit - List containing src components, used for sending traffic
dstComponent - Component used for receiving taffic
shortDesc - String, Short description, used in reporting and file prefixes
longDesc - String, Longer description, used in logging
+ Returns:
+ A string of the port id that was brought down
"""
import time
+ deltaStats = {}
+ for p in portsList:
+ deltaStats[ p ] = {}
try:
+ # Get port stats info
+ initialStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ for d in initialStats:
+ if d[ 'device' ] == device:
+ for p in d[ 'ports' ]:
+ if p[ 'port' ] in portsList:
+ deltaStats[ p[ 'port' ] ][ 'tx1' ] = p[ 'packetsSent' ]
+
main.step( "Start Capturing" )
main.funcs.startCapturing( main,
srcComponentList,
dstComponent,
shortDesc=shortDesc,
longDesc=longDesc )
+ # Let some packets flow
+ time.sleep( float( main.params['timers'].get( 'TrafficDiscovery', 5 ) ) )
+ # Get port stats info
+ updatedStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ for d in updatedStats:
+ if d[ 'device' ] == device:
+ for p in d[ 'ports' ]:
+ if p[ 'port' ] in portsList:
+ deltaStats[ p[ 'port' ] ][ 'tx2' ] = p[ 'packetsSent' ]
+ for port, stats in deltaStats.iteritems():
+ deltaStats[ port ]['delta'] = stats[ 'tx2' ] - stats[ 'tx1' ]
+
+ main.log.debug( deltaStats )
+ port = max( deltaStats, key=lambda p: deltaStats[ p ][ 'tx2' ] - deltaStats[ p ][ 'tx1' ] )
+ if deltaStats[ port ][ 'delta' ] == 0:
+ main.log.warn( "Could not find a port with traffic. Likely need to wait longer for stats to be updated" )
+ main.log.debug( port )
+ # Determine which port to bring down
main.step( "Port down" )
ctrl = main.Cluster.active( 0 ).CLI
portDown = ctrl.portstate( dpid=device, port=port, state="disable" )
@@ -254,18 +284,19 @@
adminState = p['isEnabled']
main.log.debug( adminState )
#TODO ASSERTS
- main.log.info( "Sleeping 10 seconds" )
- time.sleep(10)
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
main.step( "Stop Capturing" )
main.funcs.stopCapturing( main,
srcComponentList,
dstComponent,
shortDesc=shortDesc,
longDesc=longDesc )
+ return port
except Exception as e:
main.log.exception( "Error in linkDown" )
- def linkUp( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc ):
+ def linkUp( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
""""
High level function that handles an event including monitoring
Arguments:
@@ -277,6 +308,9 @@
longDesc - String, Longer description, used in logging
"""
import time
+ if port is None:
+ main.log.warn( "Incorrect port number, cannot bring up port" )
+ return
try:
main.step( "Start Capturing" )
main.funcs.startCapturing( main,
@@ -295,8 +329,8 @@
adminState = p['isEnabled']
main.log.debug( adminState )
#TODO ASSERTS
- main.log.info( "Sleeping 10 seconds" )
- time.sleep(10)
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
main.step( "Stop Capturing" )
main.funcs.stopCapturing( main,
srcComponentList,
@@ -387,7 +421,6 @@
main.log.warn( "Error opening " + dbFileName + " to write results." )
def cleanup( self, main ):
- # TODO: Do things like restore log levels here
run.cleanup( main )
+ main.step( "Writing csv results file for db" )
self.dbWrite( main, "SRStaging-dbfile.csv")
-