Add TOST spine failure tests
- Add SRONLReboot
- Add SRstratumRestart
- Refactoring SR Staging functions
- Use Network bench component to send traffic
- Add params file for 2x2 qa pod
- Add saving p4 write requests files
Change-Id: I60e43e2acde8b86cab0e47d62533fcf14937702d
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
index 6867bf8..16c59cc 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py
index 1f70cc1..a5724e2 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py
@@ -36,15 +36,18 @@
main.downtimeResults = {}
- # TODO: MOVE TO CONFIG FILE
- switchComponent = getattr( main, "SwitchSpine1" )
- device = "device:leaf1"
+ switchComponentList = [ getattr( main, "Spine%s" % n ) for n in range( 1, 2+1 ) ]
+ iterations = int( main.params[ 'PERF' ][ 'iterations' ] )
- ## First Spine Reboot
- shortDesc = descPrefix + "-Failure1"
- longDesc = "%s Failure: Reboot %s" % ( descPrefix, device )
- main.funcs.onlReboot( device, switchComponent, srcComponentList, dstComponent, shortDesc, longDesc )
- ## Second Spine Reboot
+ for i in range( 1, iterations + 1 ):
+ ## Spine ONL Reboot
+ shortDescFailure = descPrefix + "-Failure%s" % i
+ longDescFailure = "%s Failure%s: Reboot switch" % ( descPrefix, i )
+ shortDescRecovery = descPrefix + "-Recovery%s" % i
+ longDescRecovery = "%s Recovery%s: Reboot switch" % ( descPrefix, i )
+ main.funcs.onlReboot( switchComponentList, srcComponentList, dstComponent,
+ shortDescFailure, longDescFailure,
+ shortDescRecovery, longDescRecovery )
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
index 5e40f5a..da63514 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
@@ -25,7 +25,7 @@
</COMPONENTS>
</ONOScell>
- <SwitchLeaf1>
+ <Leaf1>
<host>10.32.4.132</host>
<user>root</user>
<password>onl</password>
@@ -41,9 +41,9 @@
<onosConfigPath></onosConfigPath>
<onosConfigFile></onosConfigFile>
</COMPONENTS>
- </SwitchLeaf1>
+ </Leaf1>
- <SwitchLeaf2>
+ <Leaf2>
<host>10.32.4.136</host>
<user>root</user>
<password>onl</password>
@@ -59,9 +59,9 @@
<onosConfigPath></onosConfigPath>
<onosConfigFile></onosConfigFile>
</COMPONENTS>
- </SwitchLeaf2>
+ </Leaf2>
- <SwitchSpine1>
+ <Spine1>
<host>10.32.4.130</host>
<user>root</user>
<password>onl</password>
@@ -77,9 +77,9 @@
<onosConfigPath></onosConfigPath>
<onosConfigFile></onosConfigFile>
</COMPONENTS>
- </SwitchSpine1>
+ </Spine1>
- <SwitchSpine2>
+ <Spine2>
<host>10.32.4.134</host>
<user>root</user>
<password>onl</password>
@@ -95,7 +95,7 @@
<onosConfigPath></onosConfigPath>
<onosConfigFile></onosConfigFile>
</COMPONENTS>
- </SwitchSpine2>
+ </Spine2>
<Host1>
<host>10.32.4.138</host>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
index 9a3590e..264f4b5 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
index 278ea9a..9079f17 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
@@ -24,6 +24,7 @@
main.funcs = SRStagingTest()
descPrefix = "eNB_Leaf_Spine_Portstate"
+ leafType = "eNB"
main.funcs.setupTest( main,
topology='2x2staging',
onosNodes=3,
@@ -38,9 +39,17 @@
dbHeaders = []
srcNames = [ src.name for src in srcComponentList ]
srcNames.sort()
- # TODO: MOVE TO CONFIG FILE
- device = "device:leaf1"
- portsList = [ 176, 180, 184, 188 ]
+ deviceShortName = None
+ portsList = []
+ for shortName, values in main.params['PERF']['topo'].iteritems():
+ if leafType in values['note']:
+ deviceShortName = shortName
+ portsList = [ int( p ) for p in values['ports'].split() ]
+ break
+ if not deviceShortName:
+ main.skipCase( result="FAIL", msg="Don't know which switch for test" )
+
+ device = "device:" + deviceShortName
port1 = None
port2 = None
port3 = None
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
index 0c91d2f..77d9899 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
index 41e1dbc..d635a91 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py
index b566254..0912afd 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py
@@ -12,4 +12,43 @@
Perform Stratum agent failure/recovery test
Collect logs and analyze results
"""
- pass
+ try:
+ from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
+ except ImportError:
+ main.log.error( "SRStagingTest not found. Exiting the test" )
+ main.cleanAndExit()
+ try:
+ main.funcs
+ except ( NameError, AttributeError ):
+ main.funcs = SRStagingTest()
+
+ descPrefix = "Stratum_Reboot"
+ main.funcs.setupTest( main,
+ topology='2x2staging',
+ onosNodes=3,
+ description="%s tests on the staging pod" % descPrefix )
+ srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
+ srcComponentList = []
+ for name in srcComponentNames:
+ srcComponentList.append( getattr( main, name ) )
+ dstComponent = getattr( main, main.params[ 'PERF' ][ 'pcap_host' ] )
+
+ main.downtimeResults = {}
+
+ switchComponentList = [ getattr( main, "Spine%s" % n ) for n in range( 1, 2+1 ) ]
+ iterations = int( main.params[ 'PERF' ][ 'iterations' ] )
+
+ for i in range( 1, iterations + 1 ):
+ ## Spine Stratum agent Reboot
+ shortDescFailure = descPrefix + "-Failure%s" % i
+ longDescFailure = "%s Failure%s: Kill Stratum on switch" % ( descPrefix, i )
+ shortDescRecovery = descPrefix + "-Recovery%s" % i
+ longDescRecovery = "%s Recovery%s: Restart Stratum on switch" % ( descPrefix, i )
+ main.funcs.killSwitchAgent( switchComponentList, srcComponentList, dstComponent,
+ shortDescFailure, longDescFailure,
+ shortDescRecovery, longDescRecovery )
+
+ main.case( "Cleanup" )
+ main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+ main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
index 478455f..11cd9ba 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
index e24f8f9..e2e5d26 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
@@ -24,6 +24,7 @@
main.funcs = SRStagingTest()
descPrefix = "Upstream_Leaf_Spine_Portstate"
+ leafType = "upstream"
main.funcs.setupTest( main,
topology='2x2staging',
onosNodes=3,
@@ -38,9 +39,17 @@
dbHeaders = []
srcNames = [ src.name for src in srcComponentList ]
srcNames.sort()
- # TODO: MOVE TO CONFIG FILE
- device = "device:leaf2"
- portsList = [260, 268, 276, 284 ]
+ deviceShortName = None
+ portsList = []
+ for shortName, values in main.params['PERF']['topo'].iteritems():
+ if leafType in values['note']:
+ deviceShortName = shortName
+ portsList = [ int( p ) for p in values['ports'].split() ]
+ break
+ if not deviceShortName:
+ main.skipCase( result="FAIL", msg="Don't know which switch for test" )
+
+ device = "device:" + deviceShortName
port1 = None
port2 = None
port3 = None
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index ad33e41..2593912 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -25,6 +25,7 @@
import json
import pexpect
+
class SRStagingTest():
def __init__( self ):
@@ -74,7 +75,7 @@
mininet_args = ' --spine=%d --leaf=%d' % ( self.topo[ topology ][ 0 ], self.topo[ topology ][ 1 ] )
if self.topo[ topology ][ 2 ]:
mininet_args += ' --dual-homed'
- if len( vlan ) > 0 :
+ if len( vlan ) > 0:
mininet_args += ' --vlan=%s' % ( ','.join( ['%d' % vlanId for vlanId in vlan ] ) )
if main.useBmv2:
mininet_args += ' --switch %s' % main.switchType
@@ -89,7 +90,9 @@
main.log.exception( "Error in setupTest" )
main.skipCase( result="FAIL", msg=e )
- def startCapturing( self, main, srcList, dst, shortDesc=None, longDesc=None ):
+ @staticmethod
+ def startCapturing( main, srcList, dst, shortDesc=None, longDesc=None,
+ trafficDuration=60, trafficSelector="-u -b 20M" ):
"""
Starts logging, traffic generation, traffic filters, etc before a failure is induced
src: the src component that sends the traffic
@@ -98,35 +101,60 @@
try:
# ping right before to make sure arp is cached and sudo is authenticated
for src in srcList:
- src.handle.sendline( "sudo /bin/ping -c 1 %s" % dst.interfaces[0]['ips'][0] )
+ src.handle.sendline( "sudo /bin/ping -w 1 -c 1 %s -I %s" % ( dst.interfaces[0]['ips'][0],
+ src.interfaces[0]['name'] ) )
try:
i = src.handle.expect( [ "password", src.prompt ] )
if i == 0:
src.handle.sendline( src.pwd )
src.handle.expect( src.prompt )
except Exception:
- main.log.error( "Unexpected response from ping" )
+ main.log.error( "%s: Unexpected response from ping" % src.name )
src.handle.send( '\x03' ) # ctrl-c
src.handle.expect( src.prompt )
+ main.funcs.clearBuffer( src )
main.log.warn( "%s: %s" % ( src.name, str( src.handle.before ) ) )
- # TODO: Create new components for iperf and tshark?
- # Also generate more streams with differnt udp ports or some other
- # method of guranteeing we kill a link with traffic
+ dst.handle.sendline( "sudo /bin/ping -w 1 -c 1 %s -I %s" % ( src.interfaces[0]['ips'][0],
+ dst.interfaces[0]['name'] ) )
+ try:
+ i = dst.handle.expect( [ "password", dst.prompt ] )
+ if i == 0:
+ dst.handle.sendline( dst.pwd )
+ dst.handle.expect( dst.prompt )
+ except Exception:
+ main.log.error( "%s: Unexpected response from ping" % dst.name )
+ dst.handle.send( '\x03' ) # ctrl-c
+ dst.handle.expect( dst.prompt )
+ main.funcs.clearBuffer( dst )
+ main.log.warn( "%s: %s" % ( dst.name, str( dst.handle.before ) ) )
# Start traffic
# TODO: ASSERTS
main.pingStart = time.time()
dstIp = dst.interfaces[0]['ips'][0]
for src in srcList:
srcIp = src.interfaces[0]['ips'][0]
- iperfArgs = "%s --bind %s -c %s" % ( main.params[ 'PERF' ][ 'traffic_cmd_arguments' ],
- srcIp,
- dstIp )
+ iperfArgs = "%s --bind %s -c %s -t %s" % ( trafficSelector,
+ srcIp,
+ dstIp,
+ trafficDuration )
main.log.info( "Starting iperf" )
- src.handle.sendline( "/usr/bin/iperf %s &> /dev/null &" % iperfArgs )
- src.handle.expect( src.prompt )
- # Check path of traffic, to use in failures
+ iperfSrc = getattr( main, "NetworkBench-%s" % src.shortName )
+ iperfSrc.handle.sendline( "sudo /bin/ping -w 1 -c 1 %s -I %s" % ( dst.interfaces[0]['ips'][0],
+ iperfSrc.interfaces[0]['name'] ) )
+ try:
+ i = iperfSrc.handle.expect( [ "password", iperfSrc.prompt ] )
+ if i == 0:
+ iperfSrc.handle.sendline( iperfSrc.pwd )
+ iperfSrc.handle.expect( iperfSrc.prompt )
+ except Exception:
+ main.log.error( "%s: Unexpected response from ping" % iperfSrc.name )
+ iperfSrc.handle.send( '\x03' ) # ctrl-c
+ iperfSrc.handle.expect( iperfSrc.prompt )
+ main.funcs.clearBuffer( iperfSrc )
+ main.log.warn( "%s: %s" % ( iperfSrc.name, str( iperfSrc.handle.before ) ) )
+
+ iperfSrc.handle.sendline( "/usr/bin/iperf %s " % iperfArgs )
# TODO: Do we need to add udp port to filter?
- # TODO: Dynamically find the interface to filter on
# Start packet capture
pcapFileReceiver = "%s/tshark/%s-%s-tsharkReceiver" % ( "~/TestON",
shortDesc if shortDesc else "tshark",
@@ -144,8 +172,7 @@
dst.handle.expect( dst.prompt )
main.log.debug( "%s: %s" % (dst.name, str( dst.handle.before ) ) )
main.log.info( "Starting tshark on %s " % dst.name )
- dst.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null &" % tsharkArgsReceiver )
- dst.handle.expect( dst.prompt )
+ dst.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null " % tsharkArgsReceiver )
for src in srcList:
srcIp = src.interfaces[0]['ips'][0]
@@ -167,9 +194,7 @@
main.log.debug( "%s: %s" % (src.name, str( src.handle.before ) ) )
main.log.info( "Starting tshark on %s " % src.name )
- for src in srcList:
- src.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null &" % tsharkArgsSender )
- src.handle.expect( src.prompt )
+ src.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null " % tsharkArgsSender )
# Timestamp used for EVENT START
main.eventStart = time.time()
# LOG Event start in ONOS logs
@@ -180,52 +205,37 @@
main.skipCase( result="FAIL", msg=e )
def stopCapturing( self, main, srcList, dst, shortDesc=None, longDesc=None ):
+ import time
try:
pcapFileReceiver = "%s/tshark/%s-%s-tsharkReceiver" % ( "~/TestON",
- shortDesc if shortDesc else "tshark",
- dst.name )
+ shortDesc if shortDesc else "tshark",
+ dst.name )
# Timestamp used for EVENT STOP
main.eventStop = time.time()
# LOG Event stop in ONOS logs
for ctrl in main.Cluster.active():
ctrl.CLI.log( "'%s STOP'" % longDesc, level="INFO" )
# Stop packet capture
- dst.handle.sendline( 'fg' ) # Bring process to front
- dst.handle.send( '\x03' ) # send ctrl-c
- try:
- for _ in range(10):
- dst.handle.expect( dst.prompt, timeout=1 )
- except pexpect.TIMEOUT:
- pass
+ main.funcs.clearBuffer( dst, kill=True, debug=True )
for src in srcList:
- src.handle.sendline( 'fg' ) # Bring process to front
- src.handle.send( '\x03' ) # send ctrl-c
- try:
- for _ in range(10):
- src.handle.expect( src.prompt, timeout=1 )
- except pexpect.TIMEOUT:
- pass
- # Stop traffic
- for src in srcList:
- src.handle.sendline( 'fg' ) # Bring process to front
- src.handle.send( '\x03' ) # send ctrl-c
- try:
- for _ in range(10):
- src.handle.expect( src.prompt, timeout=1 )
- except pexpect.TIMEOUT:
- pass
+ main.funcs.clearBuffer( src, kill=True, debug=True )
+ # Stop traffic
+ iperfSrc = getattr( main, "NetworkBench-%s" % src.shortName )
+ main.funcs.clearBuffer( iperfSrc, kill=True, debug=True )
main.pingStop = time.time()
main.log.warn( "It took %s seconds since we started ping for us to stop pcap" % ( main.pingStop - main.pingStart ) )
for src in srcList:
pcapFileSender = "%s/tshark/%s-%s-tsharkSender" % ( "~/TestON",
- shortDesc if shortDesc else "tshark",
- src.name )
+ shortDesc if shortDesc else "tshark",
+ src.name )
senderTime = self.analyzePcap( src, pcapFileSender, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False )
receiverTime = self.analyzePcap( dst, pcapFileReceiver, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False )
main.downtimeResults[ "%s-%s" % ( shortDesc, src.name ) ] = senderTime
main.downtimeResults[ "%s-%s-%s" % ( shortDesc, src.name, dst.name ) ] = receiverTime
# Grab pcap
+ # TODO: Move this elsewhere, for automatic recovery, this chould delay us
+ # to not start capture for the recovery until its already happened
senderSCP = main.ONOSbench.scp( src, pcapFileSender, main.logdir, direction="from" )
utilities.assert_equals( expect=main.TRUE, actual=senderSCP,
onpass="Saved pcap files from %s" % src.name,
@@ -237,37 +247,67 @@
onpass="Saved pcap files from %s" % dst.name,
onfail="Failed to scp pcap files from %s" % dst.name )
# Grab Write logs on switches
- # TODO: kubectl cp write-reqs.txt
+ kubeConfig = main.Cluster.active(0).k8s.kubeConfig
+ namespace = main.params[ 'kubernetes' ][ 'namespace' ]
+ switches = main.ONOSbench.kubectlGetPodNames( kubeconfig=kubeConfig,
+ namespace=namespace,
+ name="stratum" )
+ for switch in switches:
+ dstFile = "%s/%s-%s-write-reqs.txt" % ( main.logdir, shortDesc, switch )
+ writeResult = main.ONOSbench.kubectlCp( switch, "/tmp/p4_writes.txt", dstFile,
+ kubeconfig=kubeConfig,
+ namespace=namespace )
+ utilities.assert_equals( expect=main.TRUE, actual=writeResult,
+ onpass="Saved write-req file from %s" % switch,
+ onfail="Failed to cp write-req file from %s" % switch )
except Exception as e:
main.log.exception( "Error in stopCapturing" )
- def linkDown( self, device, portsList, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
+ @staticmethod
+ def clearBuffer( component, kill=False, debug=False ):
+ i = 0
+ if kill:
+ component.handle.send( "\x03" ) # Ctrl-C
+ for _ in range(10):
+ i = component.handle.expect( [ component.prompt, pexpect.TIMEOUT ] , timeout=1 )
+ if debug:
+ main.log.debug( "%s: %s" % ( component.name, str( component.handle.before ) ) )
+ if i == 1:
+ # Do we check if the ctrl-c worked?
+ break
+ """
+ if kill:
+ component.handle.send( "\x03" ) # Ctrl-C
+ component.handle.expect( component.prompt, timeout=1 )
+ """
+ try:
+ component.handle.sendline( "uname" )
+ component.handle.expect( "Linux" )
+ component.handle.expect( component.prompt )
+ except pexpect.TIMEOUT:
+ component.handle.send( "\x03" ) # Ctrl-C
+ component.handle.expect( component.prompt )
+
+ @staticmethod
+ def linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
""""
High level function that handles an event including monitoring
Arguments:
device - String of the device uri in ONOS
portsList - List of strings of the port uri in ONOS that we might take down
- srcComponentLsit - List containing src components, used for sending traffic
- dstComponent - Component used for receiving taffic
+ srcComponentList - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving traffic
shortDesc - String, Short description, used in reporting and file prefixes
longDesc - String, Longer description, used in logging
+ Option Arguments:
+ sleepTime - How long to wait between starting the capture and stopping
Returns:
A string of the port id that was brought down
"""
import time
- deltaStats = {}
- for p in portsList:
- deltaStats[ p ] = {}
try:
- # Get port stats info
initialStats = json.loads( main.Cluster.active(0).REST.portstats() )
- for d in initialStats:
- if d[ 'device' ] == device:
- for p in d[ 'ports' ]:
- if p[ 'port' ] in portsList:
- deltaStats[ p[ 'port' ] ][ 'tx1' ] = p[ 'packetsSent' ]
-
main.step( "Start Capturing" )
main.funcs.startCapturing( main,
srcComponentList,
@@ -276,26 +316,15 @@
longDesc=longDesc )
# Let some packets flow
time.sleep( float( main.params['timers'].get( 'TrafficDiscovery', 5 ) ) )
- # Get port stats info
updatedStats = json.loads( main.Cluster.active(0).REST.portstats() )
- for d in updatedStats:
- if d[ 'device' ] == device:
- for p in d[ 'ports' ]:
- if p[ 'port' ] in portsList:
- deltaStats[ p[ 'port' ] ][ 'tx2' ] = p[ 'packetsSent' ]
- for port, stats in deltaStats.iteritems():
- deltaStats[ port ]['delta'] = stats[ 'tx2' ] - stats[ 'tx1' ]
+ port = main.funcs.findPortWithTraffic( device, portsList, initialStats, updatedStats )
- main.log.debug( deltaStats )
- port = max( deltaStats, key=lambda p: deltaStats[ p ][ 'tx2' ] - deltaStats[ p ][ 'tx1' ] )
- if deltaStats[ port ][ 'delta' ] == 0:
- main.log.warn( "Could not find a port with traffic. Likely need to wait longer for stats to be updated" )
- main.log.debug( port )
# Determine which port to bring down
main.step( "Port down" )
ctrl = main.Cluster.active( 0 ).CLI
portDown = ctrl.portstate( dpid=device, port=port, state="disable" )
portsJson = json.loads( ctrl.ports() )
+ adminState = None
for d in portsJson:
if d['device']['id'] == device:
for p in d['ports']:
@@ -315,16 +344,19 @@
except Exception as e:
main.log.exception( "Error in linkDown" )
- def linkUp( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
+ @staticmethod
+ def linkUp( device, port, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
""""
High level function that handles an event including monitoring
Arguments:
device - String of the device uri in ONOS
port - String of the port uri in ONOS
- srcComponentLsit - List containing src components, used for sending traffic
- dstComponent - Component used for receiving taffic
+ srcComponentList - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving traffic
shortDesc - String, Short description, used in reporting and file prefixes
longDesc - String, Longer description, used in logging
+ Option Arguments:
+ sleepTime - How long to wait between starting the capture and stopping
"""
import time
if port is None:
@@ -341,6 +373,7 @@
ctrl = main.Cluster.active( 0 ).CLI
portUp = ctrl.portstate( dpid=device, port=port, state="enable" )
portsJson = json.loads( ctrl.ports() )
+ adminState = None
for d in portsJson:
if d['device']['id'] == device:
for p in d['ports']:
@@ -359,44 +392,228 @@
except Exception as e:
main.log.exception( "Error in linkUp" )
- def switchDown( self ):
+ @staticmethod
+ def onlReboot( switchComponentList, srcComponentList, dstComponent,
+ shortDescFailure, longDescFailure, shortDescRecovery, longDescRecovery,
+ sleepTime=5 ):
+ """"
+ High level function that handles an event including monitoring
+ Arguments:
+ switchComponent - Component used for restarting Switch
+ srcComponentList - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving traffic
+ shortDescFailure - String, Short description, used in reporting and file prefixes
+ longDescFailure - String, Longer description, used in logging
+ shortDescRecovery - String, Short description, used in reporting and file prefixes
+ longDescRecovery - String, Longer description, used in logging
+ Option Arguments:
+ sleepTime - How long to wait between starting the capture and stopping
+ """
+ import time
try:
- pass
- except Exception as e:
- main.log.exception( "Error in switchDown" )
+ main.case( longDescFailure )
+ initialStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescFailure,
+ longDesc=longDescFailure,
+ trafficDuration=120 )
+ # Let some packets flow
+ time.sleep( float( main.params['timers'].get( 'TrafficDiscovery', 5 ) ) )
+ updatedStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ switchComponent = main.funcs.findSwitchWithTraffic( switchComponentList,
+ initialStats,
+ updatedStats )
+ main.step( "Reboot ONL on Switch %s" % switchComponent.name )
+ startTime = time.time()
+ switchComponent.handle.sendline( "sudo reboot" )
- def switchUp( self ):
+ #TODO ASSERTS
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescFailure,
+ longDesc=longDescFailure )
+ # Try to minimize the amount of time we are sending 20mb/s while switch is down
+ # Large pcaps can cause timeouts when analyzing the results
+ #time.sleep( 60 )
+ main.case( longDescRecovery )
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescRecovery,
+ longDesc=longDescRecovery,
+ trafficDuration=300 )
+ # TODO: Reconnect to the NetworkBench version as well
+ connect = utilities.retry( switchComponent.connect,
+ main.FALSE,
+ attempts=30,
+ getRetryingTime=True )
+ reconnectTime = time.time()
+ main.log.warn( "It took %s seconds for the switch to reboot - ssh" % float( reconnectTime - startTime ) )
+
+ # We need to check the status of the switch in ONOS
+ available = utilities.retry( main.funcs.switchIsConnected,
+ False,
+ args=[ switchComponent ],
+ attempts=300,
+ getRetryingTime=True )
+ main.log.debug( available )
+ stopTime = time.time()
+ main.log.warn( "It took %s seconds for the switch to reconnect to ONOS" % float( stopTime - startTime ) )
+
+ main.step( "ONL Restart on Switch %s" % switchComponent.name )
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescRecovery,
+ longDesc=longDescRecovery )
+ # Check the switch is back in ONOS
+ except Exception as e:
+ main.log.exception( "Error in onlReboot" )
+
+ @staticmethod
+ def killSwitchAgent( switchComponentList, srcComponentList, dstComponent,
+ shortDescFailure, longDescFailure, shortDescRecovery, longDescRecovery,
+ sleepTime=5 ):
+ """"
+ High level function that handles an event including monitoring
+ Arguments:
+ switchComponent - Component used for restarting Switch
+ srcComponentList - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving traffic
+ shortDescFailure - String, Short description, used in reporting and file prefixes
+ longDescFailure - String, Longer description, used in logging
+ shortDescRecovery - String, Short description, used in reporting and file prefixes
+ longDescRecovery - String, Longer description, used in logging
+ Option Arguments:
+ sleepTime - How long to wait between starting the capture and stopping
+ """
+ import time
try:
- pass
- except Exception as e:
- main.log.exception( "Error in switchUp" )
+ main.case( longDescFailure )
+ initialStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescFailure,
+ longDesc=longDescFailure,
+ trafficDuration=120 )
+ # Let some packets flow
+ time.sleep( float( main.params['timers'].get( 'TrafficDiscovery', 5 ) ) )
+ updatedStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ switchComponent = main.funcs.findSwitchWithTraffic( switchComponentList,
+ initialStats,
+ updatedStats )
+ main.step( "Kill stratum agent on Switch %s" % switchComponent.name )
+ # Get pod name to delete
+ nodeName = switchComponent.shortName.lower()
+ kubeConfig = main.Cluster.active(0).k8s.kubeConfig
+ namespace = main.params[ 'kubernetes' ][ 'namespace' ]
+ output = main.ONOSbench.kubectlGetPodNames( kubeconfig=kubeConfig,
+ namespace=namespace,
+ name="stratum",
+ nodeName=nodeName )
+ main.log.debug( output )
+ if len( output ) != 1:
+ main.log.warn( "Did not find a specific switch pod to kill" )
+ startTime = time.time()
+ # Delete pod
+ main.ONOSbench.handle.sendline( "kubectl --kubeconfig %s delete pod -n %s %s" % ( kubeConfig, namespace, output[0] ) )
+ main.ONOSbench.handle.expect( main.ONOSbench.prompt )
+ main.log.debug( repr( main.ONOSbench.handle.before ) + repr( main.ONOSbench.handle.after ) )
+ #TODO ASSERTS
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescFailure,
+ longDesc=longDescFailure )
- def onosDown( self ):
+ # Try to minimize the amount of time we are sending 20mb/s while switch is down
+ #time.sleep( 60 )
+ main.case( longDescRecovery )
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescRecovery,
+ longDesc=longDescRecovery,
+ trafficDuration=400 )
+ # FIXME: We should check the health of the pod
+ #connect = utilities.retry( switchComponent.connect,
+ # main.FALSE,
+ # attempts=30,
+ # getRetryingTime=True )
+
+ # Check the status of the switch in ONOS
+ podRestartTime = time.time()
+ available = utilities.retry( main.funcs.switchIsConnected,
+ False,
+ args=[ switchComponent ],
+ attempts=300,
+ getRetryingTime=True )
+
+ main.log.debug( available )
+ stopTime = time.time()
+ main.log.warn( "It took %s seconds for the switch to reconnect to ONOS" % float( stopTime - startTime ) )
+
+ main.step( "ONL Restart on Switch %s" % switchComponent.name )
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescRecovery,
+ longDesc=longDescRecovery )
+ except Exception as e:
+ main.log.exception( "Error in killSwitchAgent" )
+
+ @staticmethod
+ def onosDown():
try:
pass
except Exception as e:
main.log.exception( "Error in onosDown" )
- def analyzePcap( self, component, filePath, packetFilter, debug=False ):
+ @staticmethod
+ def analyzePcap( component, filePath, packetFilter, debug=False, timeout=240 ):
try:
+ main.log.debug( "%s analyzing pcap file" % component.name )
+ output = ""
try:
- output = ""
component.handle.sendline( "" )
while True:
component.handle.expect( component.prompt, timeout=1 )
output += component.handle.before + str( component.handle.after )
except pexpect.TIMEOUT:
- main.log.debug( "%s: %s" % ( component.name, output ) )
+ main.log.debug( "%s analyzePcap: %s" % ( component.name, output ) )
component.handle.send( "\x03" ) # CTRL-C
- component.handle.expect( component.prompt, timeout=5 )
+ component.handle.expect( component.prompt, timeout=10 )
main.log.debug( component.handle.before + str( component.handle.after ) )
except Exception as e:
main.log.exception( "Error in onosDown" )
return -1
- lineRE = r'^\s*\d+\s+([0-9.]+)'
- tsharkOptions = "-t dd -r %s -Y %s -T fields -e frame.number -e frame.time_delta -e ip.src -e ip.dst -e udp" % ( filePath, packetFilter )
+ oneLiner = "sort -u -g -k2,2 | tail -1 | cut -f2 "
+ tsharkOptions = "-t dd -r %s -Y %s -T fields -e frame.number -e frame.time_delta -e ip.src -e ip.dst -e udp | %s" % ( filePath, packetFilter, oneLiner )
component.handle.sendline( "sudo /usr/bin/tshark %s" % tsharkOptions )
- i = component.handle.expect( [ "appears to be damaged or corrupt.", "Malformed Packet", component.prompt, pexpect.TIMEOUT ], timeout=240 )
+ i = component.handle.expect( [ "appears to be damaged or corrupt.",
+ "Malformed Packet",
+ component.prompt,
+ pexpect.TIMEOUT ], timeout=timeout )
if i != 2:
main.log.error( "Error Reading pcap file" )
main.log.debug( component.handle.before + str( component.handle.after ) )
@@ -405,6 +622,22 @@
main.log.debug( component.handle.before )
return 0
output = component.handle.before
+ assert "not found" not in output, output
+ assert "error" not in output, output
+ lineRE = r'^([0-9.]+)$'
+ for line in output.splitlines():
+ m = re.search (lineRE, line )
+ if m:
+ if debug:
+ main.log.debug( repr( line ) )
+ main.log.info( m.groups() )
+ delta = float( m.group(1) ) * 1000
+ main.log.info( "%s: Detected downtime (longest gap between packets): %s ms" % ( component.name, delta ) )
+ return delta
+ main.log.error( "No Packets found" )
+ return 0
+ """
+ lineRE = r'^\s*\d+\s+([0-9.]+)'
deltas = []
for line in output.splitlines():
# Search for a packet in each line
@@ -426,10 +659,129 @@
main.log.debug( deltas[ -10: ] ) # largest 10
main.log.info( "%s: Detected downtime (longest gap between packets): %s ms" % ( component.name, deltas[ -1 ] ) )
return deltas[ -1 ]
+ """
except Exception as e:
main.log.exception( "Error in analyzePcap" )
- def dbWrite( self, main, filename, headerOrder=None ):
+ @staticmethod
+ def findPortWithTraffic( device, portsList, initialStats, updatedStats ):
+ """
+ Given a device id and a list of ports, returns the port with the most packets sent
+ between two device statistics reads
+ Arguments:
+ device - String, device id of the device to check
+ portsList - list if ints, the ports on the device to look at
+ initialStats - A dict created from the json output of ONOS device statistics
+ updatedStats - A dict created from the json output of ONOS device statistics
+ Returns:
+ The port with the largest increase in packets sent between the two device statistics
+
+ """
+ try:
+ deltaStats = {}
+ for p in portsList:
+ deltaStats[ p ] = {}
+ for d in initialStats:
+ if d[ 'device' ] == device:
+ for p in d[ 'ports' ]:
+ if p[ 'port' ] in portsList:
+ deltaStats[ p[ 'port' ] ][ 'tx1' ] = p[ 'packetsSent' ]
+ for d in updatedStats:
+ if d[ 'device' ] == device:
+ for p in d[ 'ports' ]:
+ if p[ 'port' ] in portsList:
+ deltaStats[ p[ 'port' ] ][ 'tx2' ] = p[ 'packetsSent' ]
+ for port, stats in deltaStats.iteritems():
+ deltaStats[ port ]['delta'] = stats[ 'tx2' ] - stats[ 'tx1' ]
+ main.log.debug( deltaStats )
+ port = max( deltaStats, key=lambda p: deltaStats[ p ][ 'tx2' ] - deltaStats[ p ][ 'tx1' ] )
+ if deltaStats[ port ][ 'delta' ] == 0:
+ main.log.warn( "Could not find a port with traffic. Likely need to wait longer for stats to be updated" )
+ main.log.debug( port )
+ return port
+ except Exception as e:
+ main.log.exception( "Error in findPortWithTraffic" )
+ main.skipCase( result="FAIL", msg=e )
+
+ @staticmethod
+ def findSwitchWithTraffic( switchComponentList, initialStats, updatedStats ):
+ """
+ Given a list of switch components, returns the swtich component with the
+ port with the most packets sent between two device statistics reads
+ Arguments:
+ switchComponentList - List of switch components to check
+ initialStats - A dict created from the json output of ONOS device statistics
+ updatedStats - A dict created from the json output of ONOS device statistics
+ Returns:
+ The switch component with the port with the largest increase in packets sent
+ between the two device statistics
+
+ """
+ try:
+ deltaStats = {}
+ deviceNames = [ s.shortName for s in switchComponentList ]
+ for device in deviceNames:
+ deltaStats[ device ] = {}
+ for d in initialStats:
+ if device in d[ 'device' ]:
+ for p in d[ 'ports' ]:
+ deltaStats[ device ][ p[ 'port' ] ] = {}
+ deltaStats[ device ][ p[ 'port' ] ][ 'tx1' ] = p[ 'packetsSent' ]
+ for device in deviceNames:
+ for d in updatedStats:
+ if device in d[ 'device' ]:
+ for p in d[ 'ports' ]:
+ deltaStats[ device ][p[ 'port' ] ][ 'tx2' ] = p[ 'packetsSent' ]
+ target = ""
+ highest = 0
+ for device in deviceNames:
+ for port, stats in deltaStats[ device ].iteritems():
+ delta = stats[ 'tx2' ] - stats[ 'tx1' ]
+ if delta >= highest:
+ highest = delta
+ target = device
+ deltaStats[ device ][ port ]['delta'] = stats[ 'tx2' ] - stats[ 'tx1' ]
+
+ main.log.debug( deltaStats )
+ if highest == 0:
+ main.log.warn( "Could not find a port with traffic. Likely need to wait longer for stats to be updated" )
+ main.log.debug( target )
+ switchComponent = None
+ for switch in switchComponentList:
+ if switch.shortName is target:
+ switchComponent = switch
+ break
+ main.log.debug( switchComponent )
+ return switchComponent
+ except Exception as e:
+ main.log.exception( "Error in findSwitchWithTraffic" )
+ main.skipCase( result="FAIL", msg=e )
+
+ @staticmethod
+ def switchIsConnected( switchComponent ):
+ """
+ Given a switch component, returns if the switch is "Available" in ONOS
+ """
+ try:
+ devicesJson = json.loads( main.Cluster.active(0).devices() )
+ for device in devicesJson:
+ if switchComponent.shortName in device[ 'id' ]:
+ return device[ 'available' ]
+ return False
+ except Exception as e:
+ main.log.exception( "Error in switchIsConnected" )
+ main.skipCase( result="FAIL", msg=e )
+
+ @staticmethod
+ def dbWrite( main, filename, headerOrder=None ):
+ """
+ Prints the results stored in main.downtimeResults.
+ Arguments:
+ filename - the file name to save the results to, located in the test's log folder
+ Optional Arguments:
+ headerOrder - An ordered list of headers to write to the file. If None, will
+ print in alphabetical order
+ """
try:
dbFileName = "%s/%s" % ( main.logdir, filename )
dbfile = open( dbFileName, "w+" )