Seperate TOST Failure/Recovery tests
- Move each test to it's own file
- Each depends on the SRStaging test functions
- Topo files are symlinked to SRStaging.topo
- Add a way to specifiy db file header order or
sort them by default
- Handle issue with scp where prompt sometimes appears
before and after ouput of command
Change-Id: Ia3818285a332d0f95b2484502d4db6a0f1391211
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index e2265ce..4b803f6 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -294,11 +294,19 @@
while "to" means copy "to" the remote machine from
local machine
"""
- returnVal = main.TRUE
+ returnVal = main.FALSE
ssh_newkey = 'Are you sure you want to continue connecting'
refused = "ssh: connect to host " + \
ipAddress + " port 22: Connection refused"
cmd = "scp %s " % options
+ try:
+ self.handle.sendline( "" )
+ self.handle.expect( self.prompt, timeout=5 )
+ except pexpect.TIMEOUT:
+ main.log.error( "%s: Component not ready for input" % self.name )
+ main.log.debug( "%s: %s%s" % ( self.name, self.handle.before, str( self.handle.after ) ) )
+ self.handle.send( "\x03" ) # CTRL-C
+ self.handle.expect( self.prompt, timeout=5 )
if direction == "from":
cmd = cmd + str( userName ) + '@' + str( ipAddress ) + ':' + \
@@ -313,7 +321,8 @@
main.log.info( self.name + ": Sending: " + cmd )
self.handle.sendline( cmd )
i = 0
- while i < 2:
+ timeout = 120
+ while i <= 6 :
i = self.handle.expect( [
ssh_newkey,
'password:',
@@ -324,15 +333,17 @@
self.prompt,
pexpect.EOF,
pexpect.TIMEOUT ],
- 120 )
+ timeout=timeout )
if i == 0: # ask for ssh key confirmation
main.log.info( self.name + ": ssh key confirmation received, sending yes" )
self.handle.sendline( 'yes' )
elif i == 1: # Asked for ssh password
+ timeout = 120
main.log.info( self.name + ": ssh connection asked for password, gave password" )
self.handle.sendline( pwd )
elif i == 2: # File finished transfering
main.log.info( self.name + ": Secure copy successful" )
+ timeout = 10
returnVal = main.TRUE
elif i == 3: # Connection refused
main.log.error(
@@ -349,18 +360,21 @@
main.log.debug( self.handle.before + self.handle.after )
returnVal = main.FALSE
elif i == 6: # prompt returned
- return returnVal
+ timeout = 10
+ main.log.debug( "%s: %s%s" % ( self.name, repr( self.handle.before ), repr( self.handle.after ) ) )
elif i == 7: # EOF
main.log.error( self.name + ": Pexpect.EOF found!!!" )
main.cleanAndExit()
elif i == 8: # timeout
- main.log.error(
- "No route to the Host " +
- userName +
- "@" +
- ipAddress )
- returnVal = main.FALSE
- self.handle.expect( self.prompt )
+ if returnVal != main.TRUE:
+ main.log.error(
+ "No route to the Host " +
+ userName +
+ "@" +
+ ipAddress )
+ return returnVal
+ self.handle.expect( [ self.prompt, pexpect.TIMEOUT ], timeout=5 )
+ main.log.debug( "%s: %s%s" % ( self.name, repr( self.handle.before ), repr( self.handle.after ) ) )
return returnVal
def scp( self, remoteHost, filePath, dstPath, direction="from", options="" ):
@@ -380,7 +394,7 @@
jump_host = main.componentDictionary[ remoteHost.name ].get( 'jump_host' )
if jump_host:
jump_host = main.componentDictionary.get( jump_host )
- options += " -J %s@%s " % ( jump_host.get( 'user' ), jump_host.get( 'host' ) )
+ options += " -o 'ProxyJump %s@%s' " % ( jump_host.get( 'user' ), jump_host.get( 'host' ) )
return self.secureCopy( remoteHost.user_name,
remoteHost.ip_address,
filePath,
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
new file mode 100644
index 0000000..6867bf8
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
@@ -0,0 +1,81 @@
+<PARAMS>
+ <testcases>2</testcases>
+
+ <GRAPH>
+ <nodeCluster>staging</nodeCluster>
+ <builds>20</builds>
+ <jobName>SRONLReboot</jobName>
+ <branch>master</branch>
+ </GRAPH>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <useCommonConf>False</useCommonConf>
+ <useCommonTopo>True</useCommonTopo>
+ <useBmv2>True</useBmv2>
+ <bmv2SwitchType>stratum</bmv2SwitchType>
+ <switchPrefix></switchPrefix>
+ <stratumRoot>~/stratum</stratumRoot>
+ <topology>trellis_fabric.py</topology>
+ <lib>routinglib.py,trellislib.py,stratum.py</lib>
+ </DEPENDENCY>
+
+ <jsonFileSuffix>.hw</jsonFileSuffix>
+
+ <persistent_setup>True</persistent_setup>
+
+ <kubernetes>
+ <appName>onos-tost-onos-classic</appName>
+ <namespace>tost</namespace>
+ </kubernetes>
+
+ <PERF>
+ <traffic_host>Host1 Host2 Host3</traffic_host>
+ <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
+
+ <pcap_host>ng40vm</pcap_host>
+ <pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
+
+ </PERF>
+ <ONOS_Logging>
+ <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
+ <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
+ </ONOS_Logging>
+
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,fpm,lldpprovider,hostprovider,netcfghostprovider,drivers.bmv2,org.opencord.fabric-tofino,pipelines.fabric,org.stratumproject.fabric-tna,drivers.barefoot,segmentrouting,t3</cellApps>
+ </ENV>
+
+ <EXTERNAL_APPS>
+ </EXTERNAL_APPS>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ <TrafficDiscovery>10</TrafficDiscovery>
+ </timers>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+ <TOPO>
+ <switchNum>4</switchNum>
+ <linkNum>16</linkNum>
+ </TOPO>
+
+ <ALARM>
+ <minPassPercent>100</minPassPercent>
+ </ALARM>
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py
new file mode 100644
index 0000000..1f70cc1
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py
@@ -0,0 +1,50 @@
+class SRONLReboot:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ main.case("Testing connections")
+ main.persistentSetup = True
+
+ def CASE2( self, main ):
+ """
+ Connect to Pod
+ Perform ONL reboot failure/recovery test
+ Collect logs and analyze results
+ """
+ try:
+ from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
+ except ImportError:
+ main.log.error( "SRStagingTest not found. Exiting the test" )
+ main.cleanAndExit()
+ try:
+ main.funcs
+ except ( NameError, AttributeError ):
+ main.funcs = SRStagingTest()
+
+ descPrefix = "ONL_Reboot"
+ main.funcs.setupTest( main,
+ topology='2x2staging',
+ onosNodes=3,
+ description="%s tests on the staging pod" % descPrefix )
+ srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
+ srcComponentList = []
+ for name in srcComponentNames:
+ srcComponentList.append( getattr( main, name ) )
+ dstComponent = getattr( main, main.params[ 'PERF' ][ 'pcap_host' ] )
+
+ main.downtimeResults = {}
+
+ # TODO: MOVE TO CONFIG FILE
+ switchComponent = getattr( main, "SwitchSpine1" )
+ device = "device:leaf1"
+
+ ## First Spine Reboot
+ shortDesc = descPrefix + "-Failure1"
+ longDesc = "%s Failure: Reboot %s" % ( descPrefix, device )
+ main.funcs.onlReboot( device, switchComponent, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Second Spine Reboot
+
+ main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+ main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.topo
new file mode 120000
index 0000000..502edf5
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.topo
@@ -0,0 +1 @@
+../SRStaging.topo
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
index 2d99850..3bb151a 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>7</testcases>
+ <testcases>1</testcases>
<GRAPH>
<nodeCluster>staging</nodeCluster>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
index fd33a86..c593bbb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
@@ -183,6 +183,5 @@
longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
- main.log.warn( main.downtimeResults )
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
index 15554bb..5e40f5a 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
@@ -179,12 +179,11 @@
</Host3>
<ng40vm>
- <host>10.32.5.6</host>
+ <host>192.168.122.102</host>
<user>ng40</user>
<password>ng40</password>
<type>HostDriver</type>
<connect_order>8</connect_order>
- <jump_host>NetworkBench</jump_host>
<COMPONENTS>
<mac></mac>
<inband>false</inband>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
new file mode 100644
index 0000000..9a3590e
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
@@ -0,0 +1,81 @@
+<PARAMS>
+ <testcases>2</testcases>
+
+ <GRAPH>
+ <nodeCluster>staging</nodeCluster>
+ <builds>20</builds>
+ <jobName>SReNBLeafSpinePortstateFaliure</jobName>
+ <branch>master</branch>
+ </GRAPH>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <useCommonConf>False</useCommonConf>
+ <useCommonTopo>True</useCommonTopo>
+ <useBmv2>True</useBmv2>
+ <bmv2SwitchType>stratum</bmv2SwitchType>
+ <switchPrefix></switchPrefix>
+ <stratumRoot>~/stratum</stratumRoot>
+ <topology>trellis_fabric.py</topology>
+ <lib>routinglib.py,trellislib.py,stratum.py</lib>
+ </DEPENDENCY>
+
+ <jsonFileSuffix>.hw</jsonFileSuffix>
+
+ <persistent_setup>True</persistent_setup>
+
+ <kubernetes>
+ <appName>onos-tost-onos-classic</appName>
+ <namespace>tost</namespace>
+ </kubernetes>
+
+ <PERF>
+ <traffic_host>Host1 Host2 Host3</traffic_host>
+ <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
+
+ <pcap_host>ng40vm</pcap_host>
+ <pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
+
+ </PERF>
+ <ONOS_Logging>
+ <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
+ <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
+ </ONOS_Logging>
+
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,fpm,lldpprovider,hostprovider,netcfghostprovider,drivers.bmv2,org.opencord.fabric-tofino,pipelines.fabric,org.stratumproject.fabric-tna,drivers.barefoot,segmentrouting,t3</cellApps>
+ </ENV>
+
+ <EXTERNAL_APPS>
+ </EXTERNAL_APPS>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ <TrafficDiscovery>10</TrafficDiscovery>
+ </timers>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+ <TOPO>
+ <switchNum>4</switchNum>
+ <linkNum>16</linkNum>
+ </TOPO>
+
+ <ALARM>
+ <minPassPercent>100</minPassPercent>
+ </ALARM>
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
new file mode 100644
index 0000000..278ea9a
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
@@ -0,0 +1,115 @@
+class SReNBLeafSpinePortstateFailure:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ main.case("Testing connections")
+ main.persistentSetup = True
+
+ def CASE2( self, main ):
+ """
+ Connect to Pod
+ Perform eNB Leaf-Spine Link, portstate failure/recovery test
+ Collect logs and analyze results
+ """
+ try:
+ from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
+ except ImportError:
+ main.log.error( "SRStagingTest not found. Exiting the test" )
+ main.cleanAndExit()
+ try:
+ main.funcs
+ except ( NameError, AttributeError ):
+ main.funcs = SRStagingTest()
+
+ descPrefix = "eNB_Leaf_Spine_Portstate"
+ main.funcs.setupTest( main,
+ topology='2x2staging',
+ onosNodes=3,
+ description="%s tests on the staging pod" % descPrefix )
+ srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
+ srcComponentList = []
+ for name in srcComponentNames:
+ srcComponentList.append( getattr( main, name ) )
+ dstComponent = getattr( main, main.params[ 'PERF' ][ 'pcap_host' ] )
+
+ main.downtimeResults = {}
+ dbHeaders = []
+ srcNames = [ src.name for src in srcComponentList ]
+ srcNames.sort()
+ # TODO: MOVE TO CONFIG FILE
+ device = "device:leaf1"
+ portsList = [ 176, 180, 184, 188 ]
+ port1 = None
+ port2 = None
+ port3 = None
+ port4 = None
+
+ ## First Link Down
+ shortDesc = descPrefix + "-Failure1"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port1 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Second Link Down
+ shortDesc = descPrefix + "-Failure2"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port2 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## First Link Up
+ shortDesc = descPrefix + "-Recovery1"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port1 )
+ main.funcs.linkUp( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Second Link Up
+ shortDesc = descPrefix + "-Recovery2"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port2 )
+ main.funcs.linkUp( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Third Link Down
+ shortDesc = descPrefix + "-Failure3"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port3 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Forth Link Down
+ shortDesc = descPrefix + "-Failure4"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Third Link Up
+ shortDesc = descPrefix + "-Recovery3"
+ longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Forth Link Up
+ shortDesc = descPrefix + "-Recovery4"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+
+ main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+ main.funcs.cleanup( main, headerOrder=dbHeaders )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.topo
new file mode 120000
index 0000000..502edf5
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.topo
@@ -0,0 +1 @@
+../SRStaging.topo
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
new file mode 100644
index 0000000..0c91d2f
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
@@ -0,0 +1,81 @@
+<PARAMS>
+ <testcases>2</testcases>
+
+ <GRAPH>
+ <nodeCluster>staging</nodeCluster>
+ <builds>20</builds>
+ <jobName>SRrollingRestart</jobName>
+ <branch>master</branch>
+ </GRAPH>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <useCommonConf>False</useCommonConf>
+ <useCommonTopo>True</useCommonTopo>
+ <useBmv2>True</useBmv2>
+ <bmv2SwitchType>stratum</bmv2SwitchType>
+ <switchPrefix></switchPrefix>
+ <stratumRoot>~/stratum</stratumRoot>
+ <topology>trellis_fabric.py</topology>
+ <lib>routinglib.py,trellislib.py,stratum.py</lib>
+ </DEPENDENCY>
+
+ <jsonFileSuffix>.hw</jsonFileSuffix>
+
+ <persistent_setup>True</persistent_setup>
+
+ <kubernetes>
+ <appName>onos-tost-onos-classic</appName>
+ <namespace>tost</namespace>
+ </kubernetes>
+
+ <PERF>
+ <traffic_host>Host1 Host2 Host3</traffic_host>
+ <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
+
+ <pcap_host>ng40vm</pcap_host>
+ <pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
+
+ </PERF>
+ <ONOS_Logging>
+ <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
+ <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
+ </ONOS_Logging>
+
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,fpm,lldpprovider,hostprovider,netcfghostprovider,drivers.bmv2,org.opencord.fabric-tofino,pipelines.fabric,org.stratumproject.fabric-tna,drivers.barefoot,segmentrouting,t3</cellApps>
+ </ENV>
+
+ <EXTERNAL_APPS>
+ </EXTERNAL_APPS>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ <TrafficDiscovery>10</TrafficDiscovery>
+ </timers>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+ <TOPO>
+ <switchNum>4</switchNum>
+ <linkNum>16</linkNum>
+ </TOPO>
+
+ <ALARM>
+ <minPassPercent>100</minPassPercent>
+ </ALARM>
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.py
new file mode 100644
index 0000000..fc3c44c
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.py
@@ -0,0 +1,15 @@
+class SRrollingRestart:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ main.case("Testing connections")
+ main.persistentSetup = True
+
+ def CASE2( self, main ):
+ """
+ Connect to Pod
+ Perform rolling ONOS failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.topo
new file mode 120000
index 0000000..502edf5
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.topo
@@ -0,0 +1 @@
+../SRStaging.topo
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
new file mode 100644
index 0000000..41e1dbc
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
@@ -0,0 +1,81 @@
+<PARAMS>
+ <testcases>2</testcases>
+
+ <GRAPH>
+ <nodeCluster>staging</nodeCluster>
+ <builds>20</builds>
+ <jobName>SRstratumRestart</jobName>
+ <branch>master</branch>
+ </GRAPH>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <useCommonConf>False</useCommonConf>
+ <useCommonTopo>True</useCommonTopo>
+ <useBmv2>True</useBmv2>
+ <bmv2SwitchType>stratum</bmv2SwitchType>
+ <switchPrefix></switchPrefix>
+ <stratumRoot>~/stratum</stratumRoot>
+ <topology>trellis_fabric.py</topology>
+ <lib>routinglib.py,trellislib.py,stratum.py</lib>
+ </DEPENDENCY>
+
+ <jsonFileSuffix>.hw</jsonFileSuffix>
+
+ <persistent_setup>True</persistent_setup>
+
+ <kubernetes>
+ <appName>onos-tost-onos-classic</appName>
+ <namespace>tost</namespace>
+ </kubernetes>
+
+ <PERF>
+ <traffic_host>Host1 Host2 Host3</traffic_host>
+ <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
+
+ <pcap_host>ng40vm</pcap_host>
+ <pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
+
+ </PERF>
+ <ONOS_Logging>
+ <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
+ <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
+ </ONOS_Logging>
+
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,fpm,lldpprovider,hostprovider,netcfghostprovider,drivers.bmv2,org.opencord.fabric-tofino,pipelines.fabric,org.stratumproject.fabric-tna,drivers.barefoot,segmentrouting,t3</cellApps>
+ </ENV>
+
+ <EXTERNAL_APPS>
+ </EXTERNAL_APPS>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ <TrafficDiscovery>10</TrafficDiscovery>
+ </timers>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+ <TOPO>
+ <switchNum>4</switchNum>
+ <linkNum>16</linkNum>
+ </TOPO>
+
+ <ALARM>
+ <minPassPercent>100</minPassPercent>
+ </ALARM>
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py
new file mode 100644
index 0000000..b566254
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py
@@ -0,0 +1,15 @@
+class SRstratumRestart:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ main.case("Testing connections")
+ main.persistentSetup = True
+
+ def CASE2( self, main ):
+ """
+ Connect to Pod
+ Perform Stratum agent failure/recovery test
+ Collect logs and analyze results
+ """
+ pass
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.topo
new file mode 120000
index 0000000..502edf5
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.topo
@@ -0,0 +1 @@
+../SRStaging.topo
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
new file mode 100644
index 0000000..478455f
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
@@ -0,0 +1,81 @@
+<PARAMS>
+ <testcases>2</testcases>
+
+ <GRAPH>
+ <nodeCluster>staging</nodeCluster>
+ <builds>20</builds>
+ <jobName>SRupstreamLeafSpinePortstateFailure</jobName>
+ <branch>master</branch>
+ </GRAPH>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <useCommonConf>False</useCommonConf>
+ <useCommonTopo>True</useCommonTopo>
+ <useBmv2>True</useBmv2>
+ <bmv2SwitchType>stratum</bmv2SwitchType>
+ <switchPrefix></switchPrefix>
+ <stratumRoot>~/stratum</stratumRoot>
+ <topology>trellis_fabric.py</topology>
+ <lib>routinglib.py,trellislib.py,stratum.py</lib>
+ </DEPENDENCY>
+
+ <jsonFileSuffix>.hw</jsonFileSuffix>
+
+ <persistent_setup>True</persistent_setup>
+
+ <kubernetes>
+ <appName>onos-tost-onos-classic</appName>
+ <namespace>tost</namespace>
+ </kubernetes>
+
+ <PERF>
+ <traffic_host>Host1 Host2 Host3</traffic_host>
+ <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
+
+ <pcap_host>ng40vm</pcap_host>
+ <pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
+
+ </PERF>
+ <ONOS_Logging>
+ <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
+ <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
+ </ONOS_Logging>
+
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,fpm,lldpprovider,hostprovider,netcfghostprovider,drivers.bmv2,org.opencord.fabric-tofino,pipelines.fabric,org.stratumproject.fabric-tna,drivers.barefoot,segmentrouting,t3</cellApps>
+ </ENV>
+
+ <EXTERNAL_APPS>
+ </EXTERNAL_APPS>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ <TrafficDiscovery>10</TrafficDiscovery>
+ </timers>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+ <TOPO>
+ <switchNum>4</switchNum>
+ <linkNum>16</linkNum>
+ </TOPO>
+
+ <ALARM>
+ <minPassPercent>100</minPassPercent>
+ </ALARM>
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
new file mode 100644
index 0000000..e24f8f9
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
@@ -0,0 +1,115 @@
+class SRupstreamLeafSpinePortstateFailure:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ main.case("Testing connections")
+ main.persistentSetup = True
+
+ def CASE2( self, main ):
+ """
+ Connect to Pod
+ Perform Upstream Leaf-Spine Link, portstate failure/recovery test
+ Collect logs and analyze results
+ """
+ try:
+ from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
+ except ImportError:
+ main.log.error( "SRStagingTest not found. Exiting the test" )
+ main.cleanAndExit()
+ try:
+ main.funcs
+ except ( NameError, AttributeError ):
+ main.funcs = SRStagingTest()
+
+ descPrefix = "Upstream_Leaf_Spine_Portstate"
+ main.funcs.setupTest( main,
+ topology='2x2staging',
+ onosNodes=3,
+ description="%s tests on the staging pod" % descPrefix )
+ srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
+ srcComponentList = []
+ for name in srcComponentNames:
+ srcComponentList.append( getattr( main, name ) )
+ dstComponent = getattr( main, main.params[ 'PERF' ][ 'pcap_host' ] )
+
+ main.downtimeResults = {}
+ dbHeaders = []
+ srcNames = [ src.name for src in srcComponentList ]
+ srcNames.sort()
+ # TODO: MOVE TO CONFIG FILE
+ device = "device:leaf2"
+ portsList = [260, 268, 276, 284 ]
+ port1 = None
+ port2 = None
+ port3 = None
+ port4 = None
+
+ ## First Link Down
+ shortDesc = descPrefix + "-Failure1"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port1 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Second Link Down
+ shortDesc = descPrefix + "-Failure2"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port2 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## First Link Up
+ shortDesc = descPrefix + "-Recovery1"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port1 )
+ main.funcs.linkUp( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Second Link Up
+ shortDesc = descPrefix + "-Recovery2"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port2 )
+ main.funcs.linkUp( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Third Link Down
+ shortDesc = descPrefix + "-Failure3"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port3 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Forth Link Down
+ shortDesc = descPrefix + "-Failure4"
+ longDesc = "%s Failure: Bring down port with most traffic on %s" % ( descPrefix, device )
+ port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Third Link Up
+ shortDesc = descPrefix + "-Recovery3"
+ longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+ ## Forth Link Up
+ shortDesc = descPrefix + "-Recovery4"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
+ for src in srcNames:
+ dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
+
+ main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+ main.funcs.cleanup( main, headerOrder=dbHeaders )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.topo
new file mode 120000
index 0000000..502edf5
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.topo
@@ -0,0 +1 @@
+../SRStaging.topo
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index 50f3190..ad33e41 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -25,7 +25,7 @@
import json
import pexpect
-class SRStagingTest ():
+class SRStagingTest():
def __init__( self ):
self.default = ''
@@ -45,7 +45,7 @@
self.switchNames[ '2x2' ] = [ "leaf1", "leaf2", "spine101", "spine102" ]
main.switchType = "ovs"
- def setupTest( self, main, topology, onosNodes, description, vlan = [] ):
+ def setupTest( self, main, topology, onosNodes, description, vlan=[] ):
try:
skipPackage = False
init = False
@@ -98,7 +98,7 @@
try:
# ping right before to make sure arp is cached and sudo is authenticated
for src in srcList:
- src.handle.sendline( "sudo /bin/ping -c 1 %s" % dst.ip_address )
+ src.handle.sendline( "sudo /bin/ping -c 1 %s" % dst.interfaces[0]['ips'][0] )
try:
i = src.handle.expect( [ "password", src.prompt ] )
if i == 0:
@@ -108,7 +108,7 @@
main.log.error( "Unexpected response from ping" )
src.handle.send( '\x03' ) # ctrl-c
src.handle.expect( src.prompt )
- main.log.warn( "%s: %s" % ( src.name, src.handle.before ) )
+ main.log.warn( "%s: %s" % ( src.name, str( src.handle.before ) ) )
# TODO: Create new components for iperf and tshark?
# Also generate more streams with differnt udp ports or some other
# method of guranteeing we kill a link with traffic
@@ -142,7 +142,7 @@
for command in commands:
dst.handle.sendline( command )
dst.handle.expect( dst.prompt )
- main.log.debug( "%s: %s" % (dst.name, dst.handle.before ) )
+ main.log.debug( "%s: %s" % (dst.name, str( dst.handle.before ) ) )
main.log.info( "Starting tshark on %s " % dst.name )
dst.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null &" % tsharkArgsReceiver )
dst.handle.expect( dst.prompt )
@@ -164,7 +164,7 @@
for command in commands:
src.handle.sendline( command )
src.handle.expect( src.prompt )
- main.log.debug( "%s: %s" % (src.name, src.handle.before ) )
+ main.log.debug( "%s: %s" % (src.name, str( src.handle.before ) ) )
main.log.info( "Starting tshark on %s " % src.name )
for src in srcList:
@@ -192,31 +192,50 @@
# Stop packet capture
dst.handle.sendline( 'fg' ) # Bring process to front
dst.handle.send( '\x03' ) # send ctrl-c
- dst.handle.expect( dst.prompt )
+ try:
+ for _ in range(10):
+ dst.handle.expect( dst.prompt, timeout=1 )
+ except pexpect.TIMEOUT:
+ pass
for src in srcList:
src.handle.sendline( 'fg' ) # Bring process to front
src.handle.send( '\x03' ) # send ctrl-c
- src.handle.expect( src.prompt )
+ try:
+ for _ in range(10):
+ src.handle.expect( src.prompt, timeout=1 )
+ except pexpect.TIMEOUT:
+ pass
# Stop traffic
for src in srcList:
src.handle.sendline( 'fg' ) # Bring process to front
src.handle.send( '\x03' ) # send ctrl-c
- src.handle.expect( src.prompt )
+ try:
+ for _ in range(10):
+ src.handle.expect( src.prompt, timeout=1 )
+ except pexpect.TIMEOUT:
+ pass
main.pingStop = time.time()
main.log.warn( "It took %s seconds since we started ping for us to stop pcap" % ( main.pingStop - main.pingStart ) )
- main.downtimeResults[ shortDesc ] = {}
for src in srcList:
pcapFileSender = "%s/tshark/%s-%s-tsharkSender" % ( "~/TestON",
shortDesc if shortDesc else "tshark",
src.name )
- main.downtimeResults[ shortDesc ].update( { src.name: self.analyzePcap( src, pcapFileSender, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False) } )
- main.downtimeResults[ shortDesc ].update( { "%s-%s" % ( src.name, dst.name ): self.analyzePcap( dst, pcapFileReceiver, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False) } )
+ senderTime = self.analyzePcap( src, pcapFileSender, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False )
+ receiverTime = self.analyzePcap( dst, pcapFileReceiver, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False )
+ main.downtimeResults[ "%s-%s" % ( shortDesc, src.name ) ] = senderTime
+ main.downtimeResults[ "%s-%s-%s" % ( shortDesc, src.name, dst.name ) ] = receiverTime
# Grab pcap
senderSCP = main.ONOSbench.scp( src, pcapFileSender, main.logdir, direction="from" )
+ utilities.assert_equals( expect=main.TRUE, actual=senderSCP,
+ onpass="Saved pcap files from %s" % src.name,
+ onfail="Failed to scp pcap files from %s" % src.name )
# Grab logs
# Grab pcap
receiverSCP = main.ONOSbench.scp( dst, pcapFileReceiver, main.logdir, direction="from" )
+ utilities.assert_equals( expect=main.TRUE, actual=receiverSCP,
+ onpass="Saved pcap files from %s" % dst.name,
+ onfail="Failed to scp pcap files from %s" % dst.name )
# Grab Write logs on switches
# TODO: kubectl cp write-reqs.txt
@@ -368,16 +387,21 @@
output += component.handle.before + str( component.handle.after )
except pexpect.TIMEOUT:
main.log.debug( "%s: %s" % ( component.name, output ) )
+ component.handle.send( "\x03" ) # CTRL-C
+ component.handle.expect( component.prompt, timeout=5 )
+ main.log.debug( component.handle.before + str( component.handle.after ) )
except Exception as e:
main.log.exception( "Error in onosDown" )
+ return -1
lineRE = r'^\s*\d+\s+([0-9.]+)'
tsharkOptions = "-t dd -r %s -Y %s -T fields -e frame.number -e frame.time_delta -e ip.src -e ip.dst -e udp" % ( filePath, packetFilter )
component.handle.sendline( "sudo /usr/bin/tshark %s" % tsharkOptions )
- i = component.handle.expect( [ "appears to be damaged or corrupt.", "Malformed Packet", component.prompt, pexpect.TIMEOUT ], timeout=60 )
+ i = component.handle.expect( [ "appears to be damaged or corrupt.", "Malformed Packet", component.prompt, pexpect.TIMEOUT ], timeout=240 )
if i != 2:
main.log.error( "Error Reading pcap file" )
+ main.log.debug( component.handle.before + str( component.handle.after ) )
component.handle.send( '\x03' ) # CTRL-C to end process
- component.handle.expect( component.prompt )
+ component.handle.expect( [ component.prompt, pexpect.TIMEOUT ] )
main.log.debug( component.handle.before )
return 0
output = component.handle.before
@@ -405,22 +429,25 @@
except Exception as e:
main.log.exception( "Error in analyzePcap" )
- def dbWrite( self, main, filename ):
+ def dbWrite( self, main, filename, headerOrder=None ):
try:
dbFileName = "%s/%s" % ( main.logdir, filename )
dbfile = open( dbFileName, "w+" )
header = []
row = []
- for eventName, results in main.downtimeResults.iteritems():
- for measurementName, value in results.iteritems():
- header.append( "'%s-%s'" % ( eventName, measurementName ) )
- row.append( "'%s'" % value )
+ if not headerOrder:
+ headerOrder = main.downtimeResults.keys()
+ headerOrder.sort()
+ for item in headerOrder:
+ header.append( "'%s'" % item )
+ row.append( "'%s'" % main.downtimeResults[ item ] )
+
dbfile.write( ",".join( header ) + "\n" + ",".join( row ) + "\n" )
dbfile.close()
except IOError:
main.log.warn( "Error opening " + dbFileName + " to write results." )
- def cleanup( self, main ):
+ def cleanup( self, main, headerOrder=None ):
run.cleanup( main )
main.step( "Writing csv results file for db" )
- self.dbWrite( main, "SRStaging-dbfile.csv")
+ self.dbWrite( main, main.TEST + "-dbfile.csv", headerOrder )