[SDFAB-285] Use stern to save a tail of pod logs
- Only save logs from the test period
- Fix some typos
- Check cli connectivity after saving logs, we sometimes timed out
Change-Id: I3f62950f898bef78b6071d94bcfb17ced1a6ca45
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
index bc7af77..2562f09 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
index 1126205..2d5e5a3 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
index c593bbb..63a5280 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
@@ -102,11 +102,11 @@
port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
- longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
## Forth Link Up
shortDesc = descPrefix + "-Recovery4"
- longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
@@ -176,11 +176,11 @@
port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
- longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
## Forth Link Up
shortDesc = descPrefix + "-Recovery4"
- longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
index 9e88746..d4d6fb4 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
index 9079f17..848a8ec 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
@@ -105,7 +105,7 @@
dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
- longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
for src in srcNames:
dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
@@ -113,7 +113,7 @@
dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
## Forth Link Up
shortDesc = descPrefix + "-Recovery4"
- longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
for src in srcNames:
dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
index 0834030..11e1e74 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
@@ -26,6 +26,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -49,11 +51,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
index 40a4c6c..609d479 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
@@ -9,7 +9,7 @@
<jump_host></jump_host>
<home>~/onos</home> # defines where onos home is on the build machine. Defaults to "~/onos/" if empty.
<COMPONENTS>
- <kubeConfig>~/.kube/dev-pairedleaves-tuscon</kubeConfig> # If set, will attempt to use this file for setting up port-forwarding
+ <kubeConfig>~/.kube/dev-pairedleaves-tucson</kubeConfig> # If set, will attempt to use this file for setting up port-forwarding
<useDocker>True</useDocker> # Whether to use docker for ONOS nodes
<docker_prompt>\$</docker_prompt>
<cluster_name></cluster_name> # Used as a prefix for cluster components. Defaults to 'ONOS'
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
index 4a0b538..846e2eb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
index 9db4ffe..b1c57ad 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
index a4f7976..ba33e82 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
index e2e5d26..137a2d3 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
@@ -105,7 +105,7 @@
dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
- longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
for src in srcNames:
dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
@@ -113,7 +113,7 @@
dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
## Forth Link Up
shortDesc = descPrefix + "-Recovery4"
- longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
for src in srcNames:
dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index 4143f93..978a41e 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -231,6 +231,7 @@
receiverTime = self.analyzePcap( dst, pcapFileReceiver, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False )
main.downtimeResults[ "%s-%s" % ( shortDesc, src.name ) ] = senderTime
main.downtimeResults[ "%s-%s-%s" % ( shortDesc, src.name, dst.name ) ] = receiverTime
+ # TODO: Add alarm here if time is too high
# Grab pcap
# TODO: Move this elsewhere, for automatic recovery, this chould delay us
# to not start capture for the recovery until its already happened
@@ -239,7 +240,10 @@
onpass="Saved pcap files from %s" % src.name,
onfail="Failed to scp pcap files from %s" % src.name )
# Grab logs
- main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True, includeCaseDesc=False )
+ useStern = main.params['use_stern'].lower() == "true"
+ main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True,
+ includeCaseDesc=False, useStern=useStern,
+ startTime=main.eventStart )
# Grab pcap
receiverSCP = main.ONOSbench.scp( dst, pcapFileReceiver, main.logdir, direction="from" )
utilities.assert_equals( expect=main.TRUE, actual=receiverSCP,
@@ -849,6 +853,6 @@
main.log.warn( "Error opening " + dbFileName + " to write results." )
def cleanup( self, main, headerOrder=None ):
- run.cleanup( main )
+ run.cleanup( main, copyKarafLog=False )
main.step( "Writing csv results file for db" )
self.dbWrite( main, main.TEST + "-dbfile.csv", headerOrder )