[SDFAB-285] Use stern to save a tail of pod logs
- Only save logs from the test period
- Fix some typos
- Check cli connectivity after saving logs, we sometimes timed out
Change-Id: I3f62950f898bef78b6071d94bcfb17ced1a6ca45
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 78d0ebb..132618e 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -1115,6 +1115,56 @@
main.log.exception( self.name + ": Uncaught exception!" )
return main.FALSE
+ def sternLogs( self, podString, dstPath, kubeconfig=None, namespace=None, since='1h', wait=60 ):
+ """
+ Use stern to get the logs from a pod
+ Required Arguments:
+ - podString: The name of the pod or partial name of the pods to get the logs of
+ - dstPath: The location to save the logs to
+ Optional Arguments:
+ - kubeconfig: The path to a kubeconfig file
+ - namespace: The namespace to search in
+ - since: Return logs newer than a relative duration like 5s, 2m, or 3h. Defaults to 1h
+ - wait: How long to wait, in seconds, before killing the process. Stern does not currently
+ support a way to exit if cought up to present time. Defaults to 60 seconds
+ Returns main.TRUE or
+ main.FALSE on Error
+ """
+ import time
+ try:
+ cmdStr = "stern %s %s %s %s > %s " % (
+ "--kubeconfig %s" % kubeconfig if kubeconfig else "",
+ "-n %s" % namespace if namespace else "",
+ "--since %s" % since if since else "",
+ podString,
+ dstPath )
+ main.log.info( self.name + ": sending: " + repr( cmdStr ) )
+ self.handle.sendline( cmdStr )
+ time.sleep( int( wait ) )
+ self.handle.send( '\x03' ) # CTRL-C
+ i = self.handle.expect( [ "not found", "Error: ", "The connection to the server", self.prompt ] )
+ if i == 3:
+ main.log.debug( self.name + ": " + self.handle.before )
+ return main.TRUE
+ else:
+ main.log.error( self.name + ": Error executing command" )
+ response = self.handle.before + str( self.handle.after )
+ self.handle.expect( [ self.prompt, pexpect.TIMEOUT ], timeout=5 )
+ response += self.handle.before + str( self.handle.after )
+ main.log.debug( self.name + ": " + response )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ return main.FALSE
+
def kubectlLogs( self, podName, dstPath, kubeconfig=None, namespace=None, timeout=240 ):
"""
Use kubectl to get the logs from a pod
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
index bc7af77..2562f09 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
index 1126205..2d5e5a3 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
index c593bbb..63a5280 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
@@ -102,11 +102,11 @@
port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
- longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
## Forth Link Up
shortDesc = descPrefix + "-Recovery4"
- longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
@@ -176,11 +176,11 @@
port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
- longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
## Forth Link Up
shortDesc = descPrefix + "-Recovery4"
- longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
index 9e88746..d4d6fb4 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
index 9079f17..848a8ec 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
@@ -105,7 +105,7 @@
dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
- longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
for src in srcNames:
dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
@@ -113,7 +113,7 @@
dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
## Forth Link Up
shortDesc = descPrefix + "-Recovery4"
- longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
for src in srcNames:
dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
index 0834030..11e1e74 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
@@ -26,6 +26,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -49,11 +51,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
index 40a4c6c..609d479 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
@@ -9,7 +9,7 @@
<jump_host></jump_host>
<home>~/onos</home> # defines where onos home is on the build machine. Defaults to "~/onos/" if empty.
<COMPONENTS>
- <kubeConfig>~/.kube/dev-pairedleaves-tuscon</kubeConfig> # If set, will attempt to use this file for setting up port-forwarding
+ <kubeConfig>~/.kube/dev-pairedleaves-tucson</kubeConfig> # If set, will attempt to use this file for setting up port-forwarding
<useDocker>True</useDocker> # Whether to use docker for ONOS nodes
<docker_prompt>\$</docker_prompt>
<cluster_name></cluster_name> # Used as a prefix for cluster components. Defaults to 'ONOS'
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
index 4a0b538..846e2eb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
index 9db4ffe..b1c57ad 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
index a4f7976..ba33e82 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
@@ -28,6 +28,8 @@
<persistent_setup>True</persistent_setup>
+ <use_stern>True</use_stern>
+
<kubernetes>
<appName>onos-classic</appName>
<namespace>tost</namespace>
@@ -59,11 +61,13 @@
</PERF>
<ONOS_Logging>
<org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
</ONOS_Logging>
<ONOS_Logging_Reset>
<org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
<org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
<org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
</ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
index e2e5d26..137a2d3 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
@@ -105,7 +105,7 @@
dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
## Third Link Up
shortDesc = descPrefix + "-Recovery3"
- longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
for src in srcNames:
dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
@@ -113,7 +113,7 @@
dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
## Forth Link Up
shortDesc = descPrefix + "-Recovery4"
- longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
for src in srcNames:
dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index 4143f93..978a41e 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -231,6 +231,7 @@
receiverTime = self.analyzePcap( dst, pcapFileReceiver, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False )
main.downtimeResults[ "%s-%s" % ( shortDesc, src.name ) ] = senderTime
main.downtimeResults[ "%s-%s-%s" % ( shortDesc, src.name, dst.name ) ] = receiverTime
+ # TODO: Add alarm here if time is too high
# Grab pcap
# TODO: Move this elsewhere, for automatic recovery, this chould delay us
# to not start capture for the recovery until its already happened
@@ -239,7 +240,10 @@
onpass="Saved pcap files from %s" % src.name,
onfail="Failed to scp pcap files from %s" % src.name )
# Grab logs
- main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True, includeCaseDesc=False )
+ useStern = main.params['use_stern'].lower() == "true"
+ main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True,
+ includeCaseDesc=False, useStern=useStern,
+ startTime=main.eventStart )
# Grab pcap
receiverSCP = main.ONOSbench.scp( dst, pcapFileReceiver, main.logdir, direction="from" )
utilities.assert_equals( expect=main.TRUE, actual=receiverSCP,
@@ -849,6 +853,6 @@
main.log.warn( "Error opening " + dbFileName + " to write results." )
def cleanup( self, main, headerOrder=None ):
- run.cleanup( main )
+ run.cleanup( main, copyKarafLog=False )
main.step( "Writing csv results file for db" )
self.dbWrite( main, main.TEST + "-dbfile.csv", headerOrder )
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 5f9dd8a..bc4bea6 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -1847,6 +1847,7 @@
main.step( 'Set logging levels' )
# Get original values incase we want to reset them
ctrl = main.Cluster.active(0)
+ ctrl.CLI.clearBuffer( timeout=1 )
ctrl.CLI.logList()
logging = True
@@ -1870,6 +1871,7 @@
main.step( 'Reset logging levels' )
# Get original values incase we want to reset them
ctrl = main.Cluster.active(0)
+ ctrl.CLI.clearBuffer( timeout=1 )
currentLevels = ctrl.CLI.logList( saveValues=False )
origLevels = ctrl.CLI.logLevels
toBeSet = {}
diff --git a/TestON/tests/dependencies/utils.py b/TestON/tests/dependencies/utils.py
index 8aaca08..e787474 100644
--- a/TestON/tests/dependencies/utils.py
+++ b/TestON/tests/dependencies/utils.py
@@ -54,7 +54,8 @@
onfail="Failed to stopped mininet" )
return topoResult
- def copyKarafLog( self, copyFileName="", before=False, includeCaseDesc=True ):
+ def copyKarafLog( self, copyFileName="", before=False, includeCaseDesc=True,
+ useStern=False, startTime=None ):
"""
Description:
copy the karaf log and verify it.
@@ -63,6 +64,8 @@
copyFileName.
Returns:
"""
+ import datetime
+ import math
# TODO: Also grab the rotated karaf logs
main.log.report( "Copy karaf logs" )
if includeCaseDesc:
@@ -102,12 +105,10 @@
else:
stepResult = main.FALSE and stepResult
if isKube:
- # TODO: Look into using Stern, kail, or just use `kubectl logs <pod>`
# We also need to save the pod name to switch name mapping
main.ONOSbench.kubectlPodNodes( dstPath=main.logdir + "/podMapping.txt",
kubeconfig=ctrl.k8s.kubeConfig,
namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
- # TODO Get stratum write logs
# Save image for pods, based on "describe pods"
main.ONOSbench.kubectlDescribe( "pods",
main.logdir + "/describePods.txt",
@@ -117,12 +118,28 @@
pods = main.ONOSbench.kubectlGetPodNames( kubeconfig=ctrl.k8s.kubeConfig,
namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
+ MINUTE = datetime.timedelta( minutes=1 )
for pod in pods:
path = "%s/%s.log" % ( main.logdir, pod )
- stratumPods = main.ONOSbench.kubectlLogs( pod,
- path,
- kubeconfig=ctrl.k8s.kubeConfig,
- namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
+ if useStern:
+ if startTime:
+ now = datetime.datetime.utcnow()
+ duration = ( now - startTime ) + MINUTE
+ since = "%ss" % int( math.ceil( duration.total_seconds() ) )
+ else:
+ since = "1h"
+ podResults = main.ONOSbench.sternLogs( pod,
+ path,
+ kubeconfig=ctrl.k8s.kubeConfig,
+ namespace=main.params[ 'kubernetes' ][ 'namespace' ],
+ since=since,
+ wait=10 )
+ else:
+ podResults = main.ONOSbench.kubectlLogs( pod,
+ path,
+ kubeconfig=ctrl.k8s.kubeConfig,
+ namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
+ stepResult = stepResult and podResults
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
onpass="Successfully copied remote ONOS logs",