Add TOST spine failure tests
- Add SRONLReboot
- Add SRstratumRestart
- Refactoring SR Staging functions
- Use Network bench component to send traffic
- Add params file for 2x2 qa pod
- Add saving p4 write requests files
Change-Id: I60e43e2acde8b86cab0e47d62533fcf14937702d
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 4b803f6..2951ec3 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -985,7 +985,7 @@
print "preDisconnect"
return main.TRUE
- def kubectlGetPodNames( self, kubeconfig=None, namespace=None, app=None, name=None ):
+ def kubectlGetPodNames( self, kubeconfig=None, namespace=None, app=None, name=None, nodeName=None ):
"""
Use kubectl to get the names of pods
Optional Arguments:
@@ -993,16 +993,18 @@
- namespace: The namespace to search in
- app: Get pods belonging to a specific app
- name: Get pods with a specific name label
+ - nodeName: Get pods on a specific node
Returns a list containing the names of the pods or
main.FALSE on Error
"""
try:
- cmdStr = "kubectl %s %s get pods %s %s --output=jsonpath='{.items..metadata.name}{\"\\n\"}'" % (
+ cmdStr = "kubectl %s %s get pods %s %s %s --output=jsonpath='{.items..metadata.name}{\"\\n\"}'" % (
"--kubeconfig %s" % kubeconfig if kubeconfig else "",
"-n %s" % namespace if namespace else "",
"-l app=%s" % app if app else "",
- "-l name=%s" % name if name else "" )
+ "-l name=%s" % name if name else "",
+ "--field-selector=spec.nodeName=%s" % nodeName if nodeName else "" )
main.log.info( self.name + ": sending: " + repr( cmdStr ) )
self.handle.sendline( cmdStr )
i = self.handle.expect( [ "not found", "error", "The connection to the server", self.prompt ] )
@@ -1148,6 +1150,50 @@
main.log.exception( self.name + ": Uncaught exception!" )
return main.FALSE
+ def kubectlCp( self, podName, srcPath, dstPath, kubeconfig=None, namespace=None, timeout=240 ):
+ """
+ Use kubectl to get a file from a pod
+ Required Arguments:
+ - podName: The name of the pod to get the logs of
+ - srcPath: The file to copy from the pod
+ - dstPath: The location to save the file to locally
+ Optional Arguments:
+ - kubeconfig: The path to a kubeconfig file
+ - namespace: The namespace to search in
+ - timeout: Timeout for command to return. The longer the logs, the longer it will take to fetch them.
+ Returns main.TRUE or
+ main.FALSE on Error
+ """
+
+ try:
+ cmdStr = "kubectl %s %s cp %s:%s %s" % (
+ "--kubeconfig %s" % kubeconfig if kubeconfig else "",
+ "-n %s" % namespace if namespace else "",
+ podName,
+ srcPath,
+ dstPath )
+ main.log.info( self.name + ": sending: " + repr( cmdStr ) )
+ self.handle.sendline( cmdStr )
+ i = self.handle.expect( [ "not found", "error", "The connection to the server", self.prompt ], timeout=timeout )
+ if i == 3:
+ main.log.debug( self.name + ": " + self.handle.before )
+ return main.TRUE
+ else:
+ main.log.error( self.name + ": Error executing command" )
+ main.log.debug( self.name + ": " + self.handle.before + str( self.handle.after ) )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ return main.FALSE
+
def kubectlPortForward( self, podName, portsList, kubeconfig=None, namespace=None, ):
"""
Use kubectl to setup port forwarding from the local machine to the kubernetes pod
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.tofino b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.tofino
index 00666bc..04818d4 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.tofino
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.tofino
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1</testcases>
+ <testcases>3</testcases>
<GRAPH>
<nodeCluster>QA-Pod</nodeCluster>
@@ -34,17 +34,28 @@
</kubernetes>
<PERF>
- <traffic_host>Host3</traffic_host>
- <traffic_container>mlabbe/iperf</traffic_container>
- <traffic_container_arguments>--net=host -v /proc/net/arp:/host/arp --rm</traffic_container_arguments>
- <traffic_cmd_arguments> -u -b 20M -t 20</traffic_cmd_arguments>
-
- <pcap_host>Host4</pcap_host>
- <pcap_container>toendeavour/tshark</pcap_container>
- <pcap_container_arguments>--cap-add=NET_RAW --cap-add=NET_ADMIN --net=host --rm -v ~/TestON/tshark/:/tshark</pcap_container_arguments>
- <pcap_cmd_arguments>-t e -F pcap</pcap_cmd_arguments>
- <pcap_cmd_arguments2>-t e -F pcap</pcap_cmd_arguments2>
-
+ <traffic_host>Host2</traffic_host>
+ <pcap_host>ng40vm</pcap_host>
+ <pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>260 268 276 284</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>132 140 148 156</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
@@ -99,6 +110,7 @@
<timers>
<LinkDiscovery>12</LinkDiscovery>
<SwitchDiscovery>12</SwitchDiscovery>
+ <TrafficDiscovery>10</TrafficDiscovery>
</timers>
<SLEEP>
@@ -106,8 +118,8 @@
</SLEEP>
<TOPO>
- <switchNum>1</switchNum>
- <linkNum>0</linkNum>
+ <switchNum>4</switchNum>
+ <linkNum>16</linkNum>
</TOPO>
<ALARM>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo.2x2.physical b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo.2x2.physical
new file mode 100644
index 0000000..a236f9f
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo.2x2.physical
@@ -0,0 +1,173 @@
+<TOPOLOGY>
+ <COMPONENT>
+ <ONOScell>
+ <host>localhost</host> # ONOS "bench" machine
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosClusterDriver</type>
+ <connect_order>1</connect_order>
+ <home></home> # defines where onos home is on the build machine. Defaults to "~/onos/" if empty.
+ <COMPONENTS>
+ <kubeConfig>~/.kube/qa-ace-menlo</kubeConfig> # If set, will attempt to use this file for setting up port-forwarding
+ <useDocker>True</useDocker> # Whether to use docker for ONOS nodes
+ <docker_prompt>\$</docker_prompt>
+ <cluster_name></cluster_name> # Used as a prefix for cluster components. Defaults to 'ONOS'
+ <diff_clihost>True</diff_clihost> # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
+ <karaf_username>karaf</karaf_username>
+ <karaf_password>karaf</karaf_password>
+ <web_user>karaf</web_user>
+ <web_pass>karaf</web_pass>
+ <rest_port></rest_port>
+ <prompt></prompt> # TODO: we technically need a few of these, one per component
+ <onos_home></onos_home> # defines where onos home is on the target cell machine. Defaults to entry in "home" if empty.
+ <nodes> 7 </nodes> # number of nodes in the cluster
+ </COMPONENTS>
+ </ONOScell>
+
+ <Leaf1>
+ <host>10.128.13.209</host>
+ <user>root</user>
+ <password>onl</password>
+ <type>StratumOSSwitchDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ <prompt>#</prompt>
+ <shortName>leaf1</shortName>
+ <port1>1</port1>
+ <link1>Host1</link1>
+ <onosConfigPath></onosConfigPath>
+ <onosConfigFile></onosConfigFile>
+ </COMPONENTS>
+ </Leaf1>
+
+ <Leaf2>
+ <host>10.128.13.211</host>
+ <user>root</user>
+ <password>onl</password>
+ <type>StratumOSSwitchDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ <prompt>#</prompt>
+ <shortName>leaf2</shortName>
+ <port1>2</port1>
+ <link1>Host2</link1>
+ <onosConfigPath></onosConfigPath>
+ <onosConfigFile></onosConfigFile>
+ </COMPONENTS>
+ </Leaf2>
+
+ <Spine1>
+ <host>10.128.13.213</host>
+ <user>root</user>
+ <password>onl</password>
+ <type>StratumOSSwitchDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ <prompt>#</prompt>
+ <shortName>spine1</shortName>
+ <onosConfigPath></onosConfigPath>
+ <onosConfigFile></onosConfigFile>
+ </COMPONENTS>
+ </Spine1>
+
+ <Spine2>
+ <host>10.128.13.215</host>
+ <user>root</user>
+ <password>onl</password>
+ <type>StratumOSSwitchDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ <prompt>#</prompt>
+ <shortName>spine2</shortName>
+ <onosConfigPath></onosConfigPath>
+ <onosConfigFile></onosConfigFile>
+ </COMPONENTS>
+ </Spine2>
+
+ <Host1>
+ <host>10.128.13.253</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>HostDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ <mac>3c:fd:fe:a8:ea:30</mac>
+ <inband>false</inband>
+ <ip>192.168.102.3</ip>
+ <shortName>h1</shortName>
+ <port1>0</port1>
+ <link1>Leaf1</link1>
+ <interfaceName>ens6f0</interfaceName>
+ <routes>
+ <route1>
+ <network>192.168.101.1</network>
+ <netmask>24</netmask>
+ <gw>192.168.103.1</gw>
+ <interface></interface>
+ </route1>
+ </routes>
+ </COMPONENTS>
+ </Host1>
+
+ <Host2>
+ <host>10.128.13.253</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>HostDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ <mac>3c:fd:fe:a8:ea:31</mac>
+ <inband>false</inband>
+ <ip>192.168.103.3</ip>
+ <shortName>h2</shortName>
+ <port1>0</port1>
+ <link1>Leaf1</link1>
+ <interfaceName>ens6f1</interfaceName>
+ <routes>
+ <route1>
+ <network>192.168.101.1</network>
+ <netmask>24</netmask>
+ <gw>192.168.103.1</gw>
+ <interface></interface>
+ </route1>
+ </routes>
+ </COMPONENTS>
+ </Host2>
+
+ <ng40vm>
+ <host>10.92.1.71</host>
+ <user>ng40</user>
+ <password>ng40</password>
+ <type>HostDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS>
+ <mac>52:54:00:e3:88:36</mac>
+ <inband>false</inband>
+ <ip>192.168.101.101</ip>
+ <shortName>ng40</shortName>
+ <port1></port1>
+ <link1></link1>
+ <interfaceName>ens8</interfaceName>
+ <routes>
+ <route1>
+ <network>192.168.101.1</network>
+ <netmask>24</netmask>
+ <gw>192.168.103.1</gw>
+ <interface>ens8</interface>
+ </route1>
+ </routes>
+ </COMPONENTS>
+ </ng40vm>
+
+ <NetworkBench>
+ <host>10.128.13.253</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>NetworkDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </NetworkBench>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py b/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py
index bfba8df..be98329 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py
@@ -66,7 +66,7 @@
run.installOnos( main, skipPackage=skipPackage, cliSleep=5 )
if main.useBmv2:
switchPrefix = main.params[ 'DEPENDENCY' ].get( 'switchPrefix', '' )
- if switchPrefix is None:
+ if switchPrefix is None or "None":
switchPrefix = ''
# Translate configuration file from OVS-OFDPA to BMv2 driver
translator.bmv2ToOfdpa( main ) # Try to cleanup if switching between switch types
@@ -102,6 +102,8 @@
run.checkFlows( main, minFlowCount=self.topo[ topology ][ 5 if main.useBmv2 else 4 ] * self.topo[ topology ][ 1 ], sleep=5 )
if main.useBmv2:
switchPrefix = main.params[ 'DEPENDENCY' ].get( 'switchPrefix', '' )
+ if switchPrefix is None or "None":
+ switchPrefix = ''
if switchPrefix is not '' and switchPrefix is not None:
switchPrefix += ':'
leaf_dpid = [ "device:%sleaf%d" % ( switchPrefix, ls + 1 ) for ls in range( self.topo[ topology ][ 1 ]) ]
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
index 6867bf8..16c59cc 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py
index 1f70cc1..a5724e2 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.py
@@ -36,15 +36,18 @@
main.downtimeResults = {}
- # TODO: MOVE TO CONFIG FILE
- switchComponent = getattr( main, "SwitchSpine1" )
- device = "device:leaf1"
+ switchComponentList = [ getattr( main, "Spine%s" % n ) for n in range( 1, 2+1 ) ]
+ iterations = int( main.params[ 'PERF' ][ 'iterations' ] )
- ## First Spine Reboot
- shortDesc = descPrefix + "-Failure1"
- longDesc = "%s Failure: Reboot %s" % ( descPrefix, device )
- main.funcs.onlReboot( device, switchComponent, srcComponentList, dstComponent, shortDesc, longDesc )
- ## Second Spine Reboot
+ for i in range( 1, iterations + 1 ):
+ ## Spine ONL Reboot
+ shortDescFailure = descPrefix + "-Failure%s" % i
+ longDescFailure = "%s Failure%s: Reboot switch" % ( descPrefix, i )
+ shortDescRecovery = descPrefix + "-Recovery%s" % i
+ longDescRecovery = "%s Recovery%s: Reboot switch" % ( descPrefix, i )
+ main.funcs.onlReboot( switchComponentList, srcComponentList, dstComponent,
+ shortDescFailure, longDescFailure,
+ shortDescRecovery, longDescRecovery )
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
index 5e40f5a..da63514 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
@@ -25,7 +25,7 @@
</COMPONENTS>
</ONOScell>
- <SwitchLeaf1>
+ <Leaf1>
<host>10.32.4.132</host>
<user>root</user>
<password>onl</password>
@@ -41,9 +41,9 @@
<onosConfigPath></onosConfigPath>
<onosConfigFile></onosConfigFile>
</COMPONENTS>
- </SwitchLeaf1>
+ </Leaf1>
- <SwitchLeaf2>
+ <Leaf2>
<host>10.32.4.136</host>
<user>root</user>
<password>onl</password>
@@ -59,9 +59,9 @@
<onosConfigPath></onosConfigPath>
<onosConfigFile></onosConfigFile>
</COMPONENTS>
- </SwitchLeaf2>
+ </Leaf2>
- <SwitchSpine1>
+ <Spine1>
<host>10.32.4.130</host>
<user>root</user>
<password>onl</password>
@@ -77,9 +77,9 @@
<onosConfigPath></onosConfigPath>
<onosConfigFile></onosConfigFile>
</COMPONENTS>
- </SwitchSpine1>
+ </Spine1>
- <SwitchSpine2>
+ <Spine2>
<host>10.32.4.134</host>
<user>root</user>
<password>onl</password>
@@ -95,7 +95,7 @@
<onosConfigPath></onosConfigPath>
<onosConfigFile></onosConfigFile>
</COMPONENTS>
- </SwitchSpine2>
+ </Spine2>
<Host1>
<host>10.32.4.138</host>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
index 9a3590e..264f4b5 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
index 278ea9a..9079f17 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
@@ -24,6 +24,7 @@
main.funcs = SRStagingTest()
descPrefix = "eNB_Leaf_Spine_Portstate"
+ leafType = "eNB"
main.funcs.setupTest( main,
topology='2x2staging',
onosNodes=3,
@@ -38,9 +39,17 @@
dbHeaders = []
srcNames = [ src.name for src in srcComponentList ]
srcNames.sort()
- # TODO: MOVE TO CONFIG FILE
- device = "device:leaf1"
- portsList = [ 176, 180, 184, 188 ]
+ deviceShortName = None
+ portsList = []
+ for shortName, values in main.params['PERF']['topo'].iteritems():
+ if leafType in values['note']:
+ deviceShortName = shortName
+ portsList = [ int( p ) for p in values['ports'].split() ]
+ break
+ if not deviceShortName:
+ main.skipCase( result="FAIL", msg="Don't know which switch for test" )
+
+ device = "device:" + deviceShortName
port1 = None
port2 = None
port3 = None
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
index 0c91d2f..77d9899 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
index 41e1dbc..d635a91 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py
index b566254..0912afd 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.py
@@ -12,4 +12,43 @@
Perform Stratum agent failure/recovery test
Collect logs and analyze results
"""
- pass
+ try:
+ from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ import json
+ except ImportError:
+ main.log.error( "SRStagingTest not found. Exiting the test" )
+ main.cleanAndExit()
+ try:
+ main.funcs
+ except ( NameError, AttributeError ):
+ main.funcs = SRStagingTest()
+
+ descPrefix = "Stratum_Reboot"
+ main.funcs.setupTest( main,
+ topology='2x2staging',
+ onosNodes=3,
+ description="%s tests on the staging pod" % descPrefix )
+ srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
+ srcComponentList = []
+ for name in srcComponentNames:
+ srcComponentList.append( getattr( main, name ) )
+ dstComponent = getattr( main, main.params[ 'PERF' ][ 'pcap_host' ] )
+
+ main.downtimeResults = {}
+
+ switchComponentList = [ getattr( main, "Spine%s" % n ) for n in range( 1, 2+1 ) ]
+ iterations = int( main.params[ 'PERF' ][ 'iterations' ] )
+
+ for i in range( 1, iterations + 1 ):
+ ## Spine Stratum agent Reboot
+ shortDescFailure = descPrefix + "-Failure%s" % i
+ longDescFailure = "%s Failure%s: Kill Stratum on switch" % ( descPrefix, i )
+ shortDescRecovery = descPrefix + "-Recovery%s" % i
+ longDescRecovery = "%s Recovery%s: Restart Stratum on switch" % ( descPrefix, i )
+ main.funcs.killSwitchAgent( switchComponentList, srcComponentList, dstComponent,
+ shortDescFailure, longDescFailure,
+ shortDescRecovery, longDescRecovery )
+
+ main.case( "Cleanup" )
+ main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+ main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
index 478455f..11cd9ba 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
@@ -35,11 +35,27 @@
<PERF>
<traffic_host>Host1 Host2 Host3</traffic_host>
- <traffic_cmd_arguments> -u -b 20M -t 40</traffic_cmd_arguments>
-
<pcap_host>ng40vm</pcap_host>
<pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
-
+ <iterations>1</iterations>
+ <topo>
+ <leaf1>
+ <ports>176 180 184 188</ports>
+ <note>eNB</note>
+ </leaf1>
+ <leaf2>
+ <ports>260 268 276 284</ports>
+ <note>upstream</note>
+ </leaf2>
+ <spine1>
+ <ports>128 136 144 152</ports>
+ <note>spine</note>
+ </spine1>
+ <spine2>
+ <ports>132 140 148 156</ports>
+ <note>spine</note>
+ </spine2>
+ </topo>
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
index e24f8f9..e2e5d26 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
@@ -24,6 +24,7 @@
main.funcs = SRStagingTest()
descPrefix = "Upstream_Leaf_Spine_Portstate"
+ leafType = "upstream"
main.funcs.setupTest( main,
topology='2x2staging',
onosNodes=3,
@@ -38,9 +39,17 @@
dbHeaders = []
srcNames = [ src.name for src in srcComponentList ]
srcNames.sort()
- # TODO: MOVE TO CONFIG FILE
- device = "device:leaf2"
- portsList = [260, 268, 276, 284 ]
+ deviceShortName = None
+ portsList = []
+ for shortName, values in main.params['PERF']['topo'].iteritems():
+ if leafType in values['note']:
+ deviceShortName = shortName
+ portsList = [ int( p ) for p in values['ports'].split() ]
+ break
+ if not deviceShortName:
+ main.skipCase( result="FAIL", msg="Don't know which switch for test" )
+
+ device = "device:" + deviceShortName
port1 = None
port2 = None
port3 = None
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index ad33e41..2593912 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -25,6 +25,7 @@
import json
import pexpect
+
class SRStagingTest():
def __init__( self ):
@@ -74,7 +75,7 @@
mininet_args = ' --spine=%d --leaf=%d' % ( self.topo[ topology ][ 0 ], self.topo[ topology ][ 1 ] )
if self.topo[ topology ][ 2 ]:
mininet_args += ' --dual-homed'
- if len( vlan ) > 0 :
+ if len( vlan ) > 0:
mininet_args += ' --vlan=%s' % ( ','.join( ['%d' % vlanId for vlanId in vlan ] ) )
if main.useBmv2:
mininet_args += ' --switch %s' % main.switchType
@@ -89,7 +90,9 @@
main.log.exception( "Error in setupTest" )
main.skipCase( result="FAIL", msg=e )
- def startCapturing( self, main, srcList, dst, shortDesc=None, longDesc=None ):
+ @staticmethod
+ def startCapturing( main, srcList, dst, shortDesc=None, longDesc=None,
+ trafficDuration=60, trafficSelector="-u -b 20M" ):
"""
Starts logging, traffic generation, traffic filters, etc before a failure is induced
src: the src component that sends the traffic
@@ -98,35 +101,60 @@
try:
# ping right before to make sure arp is cached and sudo is authenticated
for src in srcList:
- src.handle.sendline( "sudo /bin/ping -c 1 %s" % dst.interfaces[0]['ips'][0] )
+ src.handle.sendline( "sudo /bin/ping -w 1 -c 1 %s -I %s" % ( dst.interfaces[0]['ips'][0],
+ src.interfaces[0]['name'] ) )
try:
i = src.handle.expect( [ "password", src.prompt ] )
if i == 0:
src.handle.sendline( src.pwd )
src.handle.expect( src.prompt )
except Exception:
- main.log.error( "Unexpected response from ping" )
+ main.log.error( "%s: Unexpected response from ping" % src.name )
src.handle.send( '\x03' ) # ctrl-c
src.handle.expect( src.prompt )
+ main.funcs.clearBuffer( src )
main.log.warn( "%s: %s" % ( src.name, str( src.handle.before ) ) )
- # TODO: Create new components for iperf and tshark?
- # Also generate more streams with differnt udp ports or some other
- # method of guranteeing we kill a link with traffic
+ dst.handle.sendline( "sudo /bin/ping -w 1 -c 1 %s -I %s" % ( src.interfaces[0]['ips'][0],
+ dst.interfaces[0]['name'] ) )
+ try:
+ i = dst.handle.expect( [ "password", dst.prompt ] )
+ if i == 0:
+ dst.handle.sendline( dst.pwd )
+ dst.handle.expect( dst.prompt )
+ except Exception:
+ main.log.error( "%s: Unexpected response from ping" % dst.name )
+ dst.handle.send( '\x03' ) # ctrl-c
+ dst.handle.expect( dst.prompt )
+ main.funcs.clearBuffer( dst )
+ main.log.warn( "%s: %s" % ( dst.name, str( dst.handle.before ) ) )
# Start traffic
# TODO: ASSERTS
main.pingStart = time.time()
dstIp = dst.interfaces[0]['ips'][0]
for src in srcList:
srcIp = src.interfaces[0]['ips'][0]
- iperfArgs = "%s --bind %s -c %s" % ( main.params[ 'PERF' ][ 'traffic_cmd_arguments' ],
- srcIp,
- dstIp )
+ iperfArgs = "%s --bind %s -c %s -t %s" % ( trafficSelector,
+ srcIp,
+ dstIp,
+ trafficDuration )
main.log.info( "Starting iperf" )
- src.handle.sendline( "/usr/bin/iperf %s &> /dev/null &" % iperfArgs )
- src.handle.expect( src.prompt )
- # Check path of traffic, to use in failures
+ iperfSrc = getattr( main, "NetworkBench-%s" % src.shortName )
+ iperfSrc.handle.sendline( "sudo /bin/ping -w 1 -c 1 %s -I %s" % ( dst.interfaces[0]['ips'][0],
+ iperfSrc.interfaces[0]['name'] ) )
+ try:
+ i = iperfSrc.handle.expect( [ "password", iperfSrc.prompt ] )
+ if i == 0:
+ iperfSrc.handle.sendline( iperfSrc.pwd )
+ iperfSrc.handle.expect( iperfSrc.prompt )
+ except Exception:
+ main.log.error( "%s: Unexpected response from ping" % iperfSrc.name )
+ iperfSrc.handle.send( '\x03' ) # ctrl-c
+ iperfSrc.handle.expect( iperfSrc.prompt )
+ main.funcs.clearBuffer( iperfSrc )
+ main.log.warn( "%s: %s" % ( iperfSrc.name, str( iperfSrc.handle.before ) ) )
+
+ iperfSrc.handle.sendline( "/usr/bin/iperf %s " % iperfArgs )
# TODO: Do we need to add udp port to filter?
- # TODO: Dynamically find the interface to filter on
# Start packet capture
pcapFileReceiver = "%s/tshark/%s-%s-tsharkReceiver" % ( "~/TestON",
shortDesc if shortDesc else "tshark",
@@ -144,8 +172,7 @@
dst.handle.expect( dst.prompt )
main.log.debug( "%s: %s" % (dst.name, str( dst.handle.before ) ) )
main.log.info( "Starting tshark on %s " % dst.name )
- dst.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null &" % tsharkArgsReceiver )
- dst.handle.expect( dst.prompt )
+ dst.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null " % tsharkArgsReceiver )
for src in srcList:
srcIp = src.interfaces[0]['ips'][0]
@@ -167,9 +194,7 @@
main.log.debug( "%s: %s" % (src.name, str( src.handle.before ) ) )
main.log.info( "Starting tshark on %s " % src.name )
- for src in srcList:
- src.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null &" % tsharkArgsSender )
- src.handle.expect( src.prompt )
+ src.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null " % tsharkArgsSender )
# Timestamp used for EVENT START
main.eventStart = time.time()
# LOG Event start in ONOS logs
@@ -180,52 +205,37 @@
main.skipCase( result="FAIL", msg=e )
def stopCapturing( self, main, srcList, dst, shortDesc=None, longDesc=None ):
+ import time
try:
pcapFileReceiver = "%s/tshark/%s-%s-tsharkReceiver" % ( "~/TestON",
- shortDesc if shortDesc else "tshark",
- dst.name )
+ shortDesc if shortDesc else "tshark",
+ dst.name )
# Timestamp used for EVENT STOP
main.eventStop = time.time()
# LOG Event stop in ONOS logs
for ctrl in main.Cluster.active():
ctrl.CLI.log( "'%s STOP'" % longDesc, level="INFO" )
# Stop packet capture
- dst.handle.sendline( 'fg' ) # Bring process to front
- dst.handle.send( '\x03' ) # send ctrl-c
- try:
- for _ in range(10):
- dst.handle.expect( dst.prompt, timeout=1 )
- except pexpect.TIMEOUT:
- pass
+ main.funcs.clearBuffer( dst, kill=True, debug=True )
for src in srcList:
- src.handle.sendline( 'fg' ) # Bring process to front
- src.handle.send( '\x03' ) # send ctrl-c
- try:
- for _ in range(10):
- src.handle.expect( src.prompt, timeout=1 )
- except pexpect.TIMEOUT:
- pass
- # Stop traffic
- for src in srcList:
- src.handle.sendline( 'fg' ) # Bring process to front
- src.handle.send( '\x03' ) # send ctrl-c
- try:
- for _ in range(10):
- src.handle.expect( src.prompt, timeout=1 )
- except pexpect.TIMEOUT:
- pass
+ main.funcs.clearBuffer( src, kill=True, debug=True )
+ # Stop traffic
+ iperfSrc = getattr( main, "NetworkBench-%s" % src.shortName )
+ main.funcs.clearBuffer( iperfSrc, kill=True, debug=True )
main.pingStop = time.time()
main.log.warn( "It took %s seconds since we started ping for us to stop pcap" % ( main.pingStop - main.pingStart ) )
for src in srcList:
pcapFileSender = "%s/tshark/%s-%s-tsharkSender" % ( "~/TestON",
- shortDesc if shortDesc else "tshark",
- src.name )
+ shortDesc if shortDesc else "tshark",
+ src.name )
senderTime = self.analyzePcap( src, pcapFileSender, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False )
receiverTime = self.analyzePcap( dst, pcapFileReceiver, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False )
main.downtimeResults[ "%s-%s" % ( shortDesc, src.name ) ] = senderTime
main.downtimeResults[ "%s-%s-%s" % ( shortDesc, src.name, dst.name ) ] = receiverTime
# Grab pcap
+ # TODO: Move this elsewhere, for automatic recovery, this chould delay us
+ # to not start capture for the recovery until its already happened
senderSCP = main.ONOSbench.scp( src, pcapFileSender, main.logdir, direction="from" )
utilities.assert_equals( expect=main.TRUE, actual=senderSCP,
onpass="Saved pcap files from %s" % src.name,
@@ -237,37 +247,67 @@
onpass="Saved pcap files from %s" % dst.name,
onfail="Failed to scp pcap files from %s" % dst.name )
# Grab Write logs on switches
- # TODO: kubectl cp write-reqs.txt
+ kubeConfig = main.Cluster.active(0).k8s.kubeConfig
+ namespace = main.params[ 'kubernetes' ][ 'namespace' ]
+ switches = main.ONOSbench.kubectlGetPodNames( kubeconfig=kubeConfig,
+ namespace=namespace,
+ name="stratum" )
+ for switch in switches:
+ dstFile = "%s/%s-%s-write-reqs.txt" % ( main.logdir, shortDesc, switch )
+ writeResult = main.ONOSbench.kubectlCp( switch, "/tmp/p4_writes.txt", dstFile,
+ kubeconfig=kubeConfig,
+ namespace=namespace )
+ utilities.assert_equals( expect=main.TRUE, actual=writeResult,
+ onpass="Saved write-req file from %s" % switch,
+ onfail="Failed to cp write-req file from %s" % switch )
except Exception as e:
main.log.exception( "Error in stopCapturing" )
- def linkDown( self, device, portsList, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
+ @staticmethod
+ def clearBuffer( component, kill=False, debug=False ):
+ i = 0
+ if kill:
+ component.handle.send( "\x03" ) # Ctrl-C
+ for _ in range(10):
+ i = component.handle.expect( [ component.prompt, pexpect.TIMEOUT ] , timeout=1 )
+ if debug:
+ main.log.debug( "%s: %s" % ( component.name, str( component.handle.before ) ) )
+ if i == 1:
+ # Do we check if the ctrl-c worked?
+ break
+ """
+ if kill:
+ component.handle.send( "\x03" ) # Ctrl-C
+ component.handle.expect( component.prompt, timeout=1 )
+ """
+ try:
+ component.handle.sendline( "uname" )
+ component.handle.expect( "Linux" )
+ component.handle.expect( component.prompt )
+ except pexpect.TIMEOUT:
+ component.handle.send( "\x03" ) # Ctrl-C
+ component.handle.expect( component.prompt )
+
+ @staticmethod
+ def linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
""""
High level function that handles an event including monitoring
Arguments:
device - String of the device uri in ONOS
portsList - List of strings of the port uri in ONOS that we might take down
- srcComponentLsit - List containing src components, used for sending traffic
- dstComponent - Component used for receiving taffic
+ srcComponentList - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving traffic
shortDesc - String, Short description, used in reporting and file prefixes
longDesc - String, Longer description, used in logging
+ Option Arguments:
+ sleepTime - How long to wait between starting the capture and stopping
Returns:
A string of the port id that was brought down
"""
import time
- deltaStats = {}
- for p in portsList:
- deltaStats[ p ] = {}
try:
- # Get port stats info
initialStats = json.loads( main.Cluster.active(0).REST.portstats() )
- for d in initialStats:
- if d[ 'device' ] == device:
- for p in d[ 'ports' ]:
- if p[ 'port' ] in portsList:
- deltaStats[ p[ 'port' ] ][ 'tx1' ] = p[ 'packetsSent' ]
-
main.step( "Start Capturing" )
main.funcs.startCapturing( main,
srcComponentList,
@@ -276,26 +316,15 @@
longDesc=longDesc )
# Let some packets flow
time.sleep( float( main.params['timers'].get( 'TrafficDiscovery', 5 ) ) )
- # Get port stats info
updatedStats = json.loads( main.Cluster.active(0).REST.portstats() )
- for d in updatedStats:
- if d[ 'device' ] == device:
- for p in d[ 'ports' ]:
- if p[ 'port' ] in portsList:
- deltaStats[ p[ 'port' ] ][ 'tx2' ] = p[ 'packetsSent' ]
- for port, stats in deltaStats.iteritems():
- deltaStats[ port ]['delta'] = stats[ 'tx2' ] - stats[ 'tx1' ]
+ port = main.funcs.findPortWithTraffic( device, portsList, initialStats, updatedStats )
- main.log.debug( deltaStats )
- port = max( deltaStats, key=lambda p: deltaStats[ p ][ 'tx2' ] - deltaStats[ p ][ 'tx1' ] )
- if deltaStats[ port ][ 'delta' ] == 0:
- main.log.warn( "Could not find a port with traffic. Likely need to wait longer for stats to be updated" )
- main.log.debug( port )
# Determine which port to bring down
main.step( "Port down" )
ctrl = main.Cluster.active( 0 ).CLI
portDown = ctrl.portstate( dpid=device, port=port, state="disable" )
portsJson = json.loads( ctrl.ports() )
+ adminState = None
for d in portsJson:
if d['device']['id'] == device:
for p in d['ports']:
@@ -315,16 +344,19 @@
except Exception as e:
main.log.exception( "Error in linkDown" )
- def linkUp( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
+ @staticmethod
+ def linkUp( device, port, srcComponentList, dstComponent, shortDesc, longDesc, sleepTime=10 ):
""""
High level function that handles an event including monitoring
Arguments:
device - String of the device uri in ONOS
port - String of the port uri in ONOS
- srcComponentLsit - List containing src components, used for sending traffic
- dstComponent - Component used for receiving taffic
+ srcComponentList - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving traffic
shortDesc - String, Short description, used in reporting and file prefixes
longDesc - String, Longer description, used in logging
+ Option Arguments:
+ sleepTime - How long to wait between starting the capture and stopping
"""
import time
if port is None:
@@ -341,6 +373,7 @@
ctrl = main.Cluster.active( 0 ).CLI
portUp = ctrl.portstate( dpid=device, port=port, state="enable" )
portsJson = json.loads( ctrl.ports() )
+ adminState = None
for d in portsJson:
if d['device']['id'] == device:
for p in d['ports']:
@@ -359,44 +392,228 @@
except Exception as e:
main.log.exception( "Error in linkUp" )
- def switchDown( self ):
+ @staticmethod
+ def onlReboot( switchComponentList, srcComponentList, dstComponent,
+ shortDescFailure, longDescFailure, shortDescRecovery, longDescRecovery,
+ sleepTime=5 ):
+ """"
+ High level function that handles an event including monitoring
+ Arguments:
+ switchComponent - Component used for restarting Switch
+ srcComponentList - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving traffic
+ shortDescFailure - String, Short description, used in reporting and file prefixes
+ longDescFailure - String, Longer description, used in logging
+ shortDescRecovery - String, Short description, used in reporting and file prefixes
+ longDescRecovery - String, Longer description, used in logging
+ Option Arguments:
+ sleepTime - How long to wait between starting the capture and stopping
+ """
+ import time
try:
- pass
- except Exception as e:
- main.log.exception( "Error in switchDown" )
+ main.case( longDescFailure )
+ initialStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescFailure,
+ longDesc=longDescFailure,
+ trafficDuration=120 )
+ # Let some packets flow
+ time.sleep( float( main.params['timers'].get( 'TrafficDiscovery', 5 ) ) )
+ updatedStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ switchComponent = main.funcs.findSwitchWithTraffic( switchComponentList,
+ initialStats,
+ updatedStats )
+ main.step( "Reboot ONL on Switch %s" % switchComponent.name )
+ startTime = time.time()
+ switchComponent.handle.sendline( "sudo reboot" )
- def switchUp( self ):
+ #TODO ASSERTS
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescFailure,
+ longDesc=longDescFailure )
+ # Try to minimize the amount of time we are sending 20mb/s while switch is down
+ # Large pcaps can cause timeouts when analyzing the results
+ #time.sleep( 60 )
+ main.case( longDescRecovery )
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescRecovery,
+ longDesc=longDescRecovery,
+ trafficDuration=300 )
+ # TODO: Reconnect to the NetworkBench version as well
+ connect = utilities.retry( switchComponent.connect,
+ main.FALSE,
+ attempts=30,
+ getRetryingTime=True )
+ reconnectTime = time.time()
+ main.log.warn( "It took %s seconds for the switch to reboot - ssh" % float( reconnectTime - startTime ) )
+
+ # We need to check the status of the switch in ONOS
+ available = utilities.retry( main.funcs.switchIsConnected,
+ False,
+ args=[ switchComponent ],
+ attempts=300,
+ getRetryingTime=True )
+ main.log.debug( available )
+ stopTime = time.time()
+ main.log.warn( "It took %s seconds for the switch to reconnect to ONOS" % float( stopTime - startTime ) )
+
+ main.step( "ONL Restart on Switch %s" % switchComponent.name )
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescRecovery,
+ longDesc=longDescRecovery )
+ # Check the switch is back in ONOS
+ except Exception as e:
+ main.log.exception( "Error in onlReboot" )
+
+ @staticmethod
+ def killSwitchAgent( switchComponentList, srcComponentList, dstComponent,
+ shortDescFailure, longDescFailure, shortDescRecovery, longDescRecovery,
+ sleepTime=5 ):
+ """"
+ High level function that handles an event including monitoring
+ Arguments:
+ switchComponent - Component used for restarting Switch
+ srcComponentList - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving traffic
+ shortDescFailure - String, Short description, used in reporting and file prefixes
+ longDescFailure - String, Longer description, used in logging
+ shortDescRecovery - String, Short description, used in reporting and file prefixes
+ longDescRecovery - String, Longer description, used in logging
+ Option Arguments:
+ sleepTime - How long to wait between starting the capture and stopping
+ """
+ import time
try:
- pass
- except Exception as e:
- main.log.exception( "Error in switchUp" )
+ main.case( longDescFailure )
+ initialStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescFailure,
+ longDesc=longDescFailure,
+ trafficDuration=120 )
+ # Let some packets flow
+ time.sleep( float( main.params['timers'].get( 'TrafficDiscovery', 5 ) ) )
+ updatedStats = json.loads( main.Cluster.active(0).REST.portstats() )
+ switchComponent = main.funcs.findSwitchWithTraffic( switchComponentList,
+ initialStats,
+ updatedStats )
+ main.step( "Kill stratum agent on Switch %s" % switchComponent.name )
+ # Get pod name to delete
+ nodeName = switchComponent.shortName.lower()
+ kubeConfig = main.Cluster.active(0).k8s.kubeConfig
+ namespace = main.params[ 'kubernetes' ][ 'namespace' ]
+ output = main.ONOSbench.kubectlGetPodNames( kubeconfig=kubeConfig,
+ namespace=namespace,
+ name="stratum",
+ nodeName=nodeName )
+ main.log.debug( output )
+ if len( output ) != 1:
+ main.log.warn( "Did not find a specific switch pod to kill" )
+ startTime = time.time()
+ # Delete pod
+ main.ONOSbench.handle.sendline( "kubectl --kubeconfig %s delete pod -n %s %s" % ( kubeConfig, namespace, output[0] ) )
+ main.ONOSbench.handle.expect( main.ONOSbench.prompt )
+ main.log.debug( repr( main.ONOSbench.handle.before ) + repr( main.ONOSbench.handle.after ) )
+ #TODO ASSERTS
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescFailure,
+ longDesc=longDescFailure )
- def onosDown( self ):
+ # Try to minimize the amount of time we are sending 20mb/s while switch is down
+ #time.sleep( 60 )
+ main.case( longDescRecovery )
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescRecovery,
+ longDesc=longDescRecovery,
+ trafficDuration=400 )
+ # FIXME: We should check the health of the pod
+ #connect = utilities.retry( switchComponent.connect,
+ # main.FALSE,
+ # attempts=30,
+ # getRetryingTime=True )
+
+ # Check the status of the switch in ONOS
+ podRestartTime = time.time()
+ available = utilities.retry( main.funcs.switchIsConnected,
+ False,
+ args=[ switchComponent ],
+ attempts=300,
+ getRetryingTime=True )
+
+ main.log.debug( available )
+ stopTime = time.time()
+ main.log.warn( "It took %s seconds for the switch to reconnect to ONOS" % float( stopTime - startTime ) )
+
+ main.step( "ONL Restart on Switch %s" % switchComponent.name )
+ main.log.info( "Sleeping %s seconds" % sleepTime )
+ time.sleep( sleepTime )
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDescRecovery,
+ longDesc=longDescRecovery )
+ except Exception as e:
+ main.log.exception( "Error in killSwitchAgent" )
+
+ @staticmethod
+ def onosDown():
try:
pass
except Exception as e:
main.log.exception( "Error in onosDown" )
- def analyzePcap( self, component, filePath, packetFilter, debug=False ):
+ @staticmethod
+ def analyzePcap( component, filePath, packetFilter, debug=False, timeout=240 ):
try:
+ main.log.debug( "%s analyzing pcap file" % component.name )
+ output = ""
try:
- output = ""
component.handle.sendline( "" )
while True:
component.handle.expect( component.prompt, timeout=1 )
output += component.handle.before + str( component.handle.after )
except pexpect.TIMEOUT:
- main.log.debug( "%s: %s" % ( component.name, output ) )
+ main.log.debug( "%s analyzePcap: %s" % ( component.name, output ) )
component.handle.send( "\x03" ) # CTRL-C
- component.handle.expect( component.prompt, timeout=5 )
+ component.handle.expect( component.prompt, timeout=10 )
main.log.debug( component.handle.before + str( component.handle.after ) )
except Exception as e:
main.log.exception( "Error in onosDown" )
return -1
- lineRE = r'^\s*\d+\s+([0-9.]+)'
- tsharkOptions = "-t dd -r %s -Y %s -T fields -e frame.number -e frame.time_delta -e ip.src -e ip.dst -e udp" % ( filePath, packetFilter )
+ oneLiner = "sort -u -g -k2,2 | tail -1 | cut -f2 "
+ tsharkOptions = "-t dd -r %s -Y %s -T fields -e frame.number -e frame.time_delta -e ip.src -e ip.dst -e udp | %s" % ( filePath, packetFilter, oneLiner )
component.handle.sendline( "sudo /usr/bin/tshark %s" % tsharkOptions )
- i = component.handle.expect( [ "appears to be damaged or corrupt.", "Malformed Packet", component.prompt, pexpect.TIMEOUT ], timeout=240 )
+ i = component.handle.expect( [ "appears to be damaged or corrupt.",
+ "Malformed Packet",
+ component.prompt,
+ pexpect.TIMEOUT ], timeout=timeout )
if i != 2:
main.log.error( "Error Reading pcap file" )
main.log.debug( component.handle.before + str( component.handle.after ) )
@@ -405,6 +622,22 @@
main.log.debug( component.handle.before )
return 0
output = component.handle.before
+ assert "not found" not in output, output
+ assert "error" not in output, output
+ lineRE = r'^([0-9.]+)$'
+ for line in output.splitlines():
+ m = re.search (lineRE, line )
+ if m:
+ if debug:
+ main.log.debug( repr( line ) )
+ main.log.info( m.groups() )
+ delta = float( m.group(1) ) * 1000
+ main.log.info( "%s: Detected downtime (longest gap between packets): %s ms" % ( component.name, delta ) )
+ return delta
+ main.log.error( "No Packets found" )
+ return 0
+ """
+ lineRE = r'^\s*\d+\s+([0-9.]+)'
deltas = []
for line in output.splitlines():
# Search for a packet in each line
@@ -426,10 +659,129 @@
main.log.debug( deltas[ -10: ] ) # largest 10
main.log.info( "%s: Detected downtime (longest gap between packets): %s ms" % ( component.name, deltas[ -1 ] ) )
return deltas[ -1 ]
+ """
except Exception as e:
main.log.exception( "Error in analyzePcap" )
- def dbWrite( self, main, filename, headerOrder=None ):
+ @staticmethod
+ def findPortWithTraffic( device, portsList, initialStats, updatedStats ):
+ """
+ Given a device id and a list of ports, returns the port with the most packets sent
+ between two device statistics reads
+ Arguments:
+ device - String, device id of the device to check
+ portsList - list if ints, the ports on the device to look at
+ initialStats - A dict created from the json output of ONOS device statistics
+ updatedStats - A dict created from the json output of ONOS device statistics
+ Returns:
+ The port with the largest increase in packets sent between the two device statistics
+
+ """
+ try:
+ deltaStats = {}
+ for p in portsList:
+ deltaStats[ p ] = {}
+ for d in initialStats:
+ if d[ 'device' ] == device:
+ for p in d[ 'ports' ]:
+ if p[ 'port' ] in portsList:
+ deltaStats[ p[ 'port' ] ][ 'tx1' ] = p[ 'packetsSent' ]
+ for d in updatedStats:
+ if d[ 'device' ] == device:
+ for p in d[ 'ports' ]:
+ if p[ 'port' ] in portsList:
+ deltaStats[ p[ 'port' ] ][ 'tx2' ] = p[ 'packetsSent' ]
+ for port, stats in deltaStats.iteritems():
+ deltaStats[ port ]['delta'] = stats[ 'tx2' ] - stats[ 'tx1' ]
+ main.log.debug( deltaStats )
+ port = max( deltaStats, key=lambda p: deltaStats[ p ][ 'tx2' ] - deltaStats[ p ][ 'tx1' ] )
+ if deltaStats[ port ][ 'delta' ] == 0:
+ main.log.warn( "Could not find a port with traffic. Likely need to wait longer for stats to be updated" )
+ main.log.debug( port )
+ return port
+ except Exception as e:
+ main.log.exception( "Error in findPortWithTraffic" )
+ main.skipCase( result="FAIL", msg=e )
+
+ @staticmethod
+ def findSwitchWithTraffic( switchComponentList, initialStats, updatedStats ):
+ """
+ Given a list of switch components, returns the swtich component with the
+ port with the most packets sent between two device statistics reads
+ Arguments:
+ switchComponentList - List of switch components to check
+ initialStats - A dict created from the json output of ONOS device statistics
+ updatedStats - A dict created from the json output of ONOS device statistics
+ Returns:
+ The switch component with the port with the largest increase in packets sent
+ between the two device statistics
+
+ """
+ try:
+ deltaStats = {}
+ deviceNames = [ s.shortName for s in switchComponentList ]
+ for device in deviceNames:
+ deltaStats[ device ] = {}
+ for d in initialStats:
+ if device in d[ 'device' ]:
+ for p in d[ 'ports' ]:
+ deltaStats[ device ][ p[ 'port' ] ] = {}
+ deltaStats[ device ][ p[ 'port' ] ][ 'tx1' ] = p[ 'packetsSent' ]
+ for device in deviceNames:
+ for d in updatedStats:
+ if device in d[ 'device' ]:
+ for p in d[ 'ports' ]:
+ deltaStats[ device ][p[ 'port' ] ][ 'tx2' ] = p[ 'packetsSent' ]
+ target = ""
+ highest = 0
+ for device in deviceNames:
+ for port, stats in deltaStats[ device ].iteritems():
+ delta = stats[ 'tx2' ] - stats[ 'tx1' ]
+ if delta >= highest:
+ highest = delta
+ target = device
+ deltaStats[ device ][ port ]['delta'] = stats[ 'tx2' ] - stats[ 'tx1' ]
+
+ main.log.debug( deltaStats )
+ if highest == 0:
+ main.log.warn( "Could not find a port with traffic. Likely need to wait longer for stats to be updated" )
+ main.log.debug( target )
+ switchComponent = None
+ for switch in switchComponentList:
+ if switch.shortName is target:
+ switchComponent = switch
+ break
+ main.log.debug( switchComponent )
+ return switchComponent
+ except Exception as e:
+ main.log.exception( "Error in findSwitchWithTraffic" )
+ main.skipCase( result="FAIL", msg=e )
+
+ @staticmethod
+ def switchIsConnected( switchComponent ):
+ """
+ Given a switch component, returns if the switch is "Available" in ONOS
+ """
+ try:
+ devicesJson = json.loads( main.Cluster.active(0).devices() )
+ for device in devicesJson:
+ if switchComponent.shortName in device[ 'id' ]:
+ return device[ 'available' ]
+ return False
+ except Exception as e:
+ main.log.exception( "Error in switchIsConnected" )
+ main.skipCase( result="FAIL", msg=e )
+
+ @staticmethod
+ def dbWrite( main, filename, headerOrder=None ):
+ """
+ Prints the results stored in main.downtimeResults.
+ Arguments:
+ filename - the file name to save the results to, located in the test's log folder
+ Optional Arguments:
+ headerOrder - An ordered list of headers to write to the file. If None, will
+ print in alphabetical order
+ """
try:
dbFileName = "%s/%s" % ( main.logdir, filename )
dbfile = open( dbFileName, "w+" )