[SDFAB-285] Use stern to save a tail of pod logs

 - Only save logs from the test period
 - Fix some typos
 - Check cli connectivity after saving logs, we sometimes timed out

Change-Id: I3f62950f898bef78b6071d94bcfb17ced1a6ca45
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
index bc7af77..2562f09 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRONLReboot/SRONLReboot.params
@@ -28,6 +28,8 @@
 
     <persistent_setup>True</persistent_setup>
 
+    <use_stern>True</use_stern>
+
     <kubernetes>
         <appName>onos-classic</appName>
         <namespace>tost</namespace>
@@ -59,11 +61,13 @@
     </PERF>
     <ONOS_Logging>
         <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
     </ONOS_Logging>
     <ONOS_Logging_Reset>
         <org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
     </ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
index 1126205..2d5e5a3 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
@@ -28,6 +28,8 @@
 
     <persistent_setup>True</persistent_setup>
 
+    <use_stern>True</use_stern>
+
     <kubernetes>
         <appName>onos-classic</appName>
         <namespace>tost</namespace>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
index c593bbb..63a5280 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
@@ -102,11 +102,11 @@
         port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
         ## Third Link Up
         shortDesc = descPrefix + "-Recovery3"
-        longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+        longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
         main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
         ## Forth Link Up
         shortDesc = descPrefix + "-Recovery4"
-        longDesc = "%s Recovery: Bring up  %s/%s" % ( descPrefix, device, port4 )
+        longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
         main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
 
         main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
@@ -176,11 +176,11 @@
         port4 = main.funcs.linkDown( device, portsList, srcComponentList, dstComponent, shortDesc, longDesc )
         ## Third Link Up
         shortDesc = descPrefix + "-Recovery3"
-        longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+        longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
         main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
         ## Forth Link Up
         shortDesc = descPrefix + "-Recovery4"
-        longDesc = "%s Recovery: Bring up  %s/%s" % ( descPrefix, device, port4 )
+        longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
         main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
 
         main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
index 9e88746..d4d6fb4 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.params
@@ -28,6 +28,8 @@
 
     <persistent_setup>True</persistent_setup>
 
+    <use_stern>True</use_stern>
+
     <kubernetes>
         <appName>onos-classic</appName>
         <namespace>tost</namespace>
@@ -59,11 +61,13 @@
     </PERF>
     <ONOS_Logging>
         <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
     </ONOS_Logging>
     <ONOS_Logging_Reset>
         <org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
     </ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
index 9079f17..848a8ec 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SReNBLeafSpinePortstateFailure/SReNBLeafSpinePortstateFailure.py
@@ -105,7 +105,7 @@
             dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
         ## Third Link Up
         shortDesc = descPrefix + "-Recovery3"
-        longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+        longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
         main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
         for src in srcNames:
             dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
@@ -113,7 +113,7 @@
             dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
         ## Forth Link Up
         shortDesc = descPrefix + "-Recovery4"
-        longDesc = "%s Recovery: Bring up  %s/%s" % ( descPrefix, device, port4 )
+        longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
         main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
         for src in srcNames:
             dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
index 0834030..11e1e74 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.params
@@ -26,6 +26,8 @@
 
     <persistent_setup>True</persistent_setup>
 
+    <use_stern>True</use_stern>
+
     <kubernetes>
         <appName>onos-classic</appName>
         <namespace>tost</namespace>
@@ -49,11 +51,13 @@
     </PERF>
     <ONOS_Logging>
         <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
     </ONOS_Logging>
     <ONOS_Logging_Reset>
         <org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
     </ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
index 40a4c6c..609d479 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.topo
@@ -9,7 +9,7 @@
             <jump_host></jump_host>
             <home>~/onos</home>   # defines where onos home is on the build machine. Defaults to "~/onos/" if empty.
             <COMPONENTS>
-                <kubeConfig>~/.kube/dev-pairedleaves-tuscon</kubeConfig>  # If set, will attempt to use this file for setting up port-forwarding
+                <kubeConfig>~/.kube/dev-pairedleaves-tucson</kubeConfig>  # If set, will attempt to use this file for setting up port-forwarding
                 <useDocker>True</useDocker>  # Whether to use docker for ONOS nodes
                 <docker_prompt>\$</docker_prompt>
                 <cluster_name></cluster_name>  # Used as a prefix for cluster components. Defaults to 'ONOS'
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
index 4a0b538..846e2eb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRrollingRestart/SRrollingRestart.params
@@ -28,6 +28,8 @@
 
     <persistent_setup>True</persistent_setup>
 
+    <use_stern>True</use_stern>
+
     <kubernetes>
         <appName>onos-classic</appName>
         <namespace>tost</namespace>
@@ -59,11 +61,13 @@
     </PERF>
     <ONOS_Logging>
         <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
     </ONOS_Logging>
     <ONOS_Logging_Reset>
         <org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
     </ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
index 9db4ffe..b1c57ad 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRstratumRestart/SRstratumRestart.params
@@ -28,6 +28,8 @@
 
     <persistent_setup>True</persistent_setup>
 
+    <use_stern>True</use_stern>
+
     <kubernetes>
         <appName>onos-classic</appName>
         <namespace>tost</namespace>
@@ -59,11 +61,13 @@
     </PERF>
     <ONOS_Logging>
         <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
     </ONOS_Logging>
     <ONOS_Logging_Reset>
         <org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
     </ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
index a4f7976..ba33e82 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.params
@@ -28,6 +28,8 @@
 
     <persistent_setup>True</persistent_setup>
 
+    <use_stern>True</use_stern>
+
     <kubernetes>
         <appName>onos-classic</appName>
         <namespace>tost</namespace>
@@ -59,11 +61,13 @@
     </PERF>
     <ONOS_Logging>
         <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>TRACE</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
     </ONOS_Logging>
     <ONOS_Logging_Reset>
         <org.onosproject.p4runtime.ctl.client>INFO</org.onosproject.p4runtime.ctl.client>
+        <org.onosproject.p4runtime.ctl.client.writerequestimpl>INFO</org.onosproject.p4runtime.ctl.client.writerequestimpl>
         <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
         <org.onosproject.gnmi.ctl>INFO</org.onosproject.gnmi.ctl>
     </ONOS_Logging_Reset>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
index e2e5d26..137a2d3 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRupstreamLeafSpinePortstateFailure/SRupstreamLeafSpinePortstateFailure.py
@@ -105,7 +105,7 @@
             dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
         ## Third Link Up
         shortDesc = descPrefix + "-Recovery3"
-        longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+        longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port3 )
         main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
         for src in srcNames:
             dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
@@ -113,7 +113,7 @@
             dbHeaders.append( "%s-%s-%s" % ( shortDesc, src, dstComponent.name ) )
         ## Forth Link Up
         shortDesc = descPrefix + "-Recovery4"
-        longDesc = "%s Recovery: Bring up  %s/%s" % ( descPrefix, device, port4 )
+        longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
         main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
         for src in srcNames:
             dbHeaders.append( "%s-%s" % ( shortDesc, src ) )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index 4143f93..978a41e 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -231,6 +231,7 @@
                 receiverTime = self.analyzePcap( dst, pcapFileReceiver, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False )
                 main.downtimeResults[ "%s-%s" % ( shortDesc, src.name ) ] = senderTime
                 main.downtimeResults[ "%s-%s-%s" % ( shortDesc, src.name, dst.name ) ] = receiverTime
+                # TODO: Add alarm here if time is too high
                 # Grab pcap
                 # TODO: Move this elsewhere, for automatic recovery, this chould delay us
                 #       to not start capture for the recovery until its already happened
@@ -239,7 +240,10 @@
                                          onpass="Saved pcap files from %s" % src.name,
                                          onfail="Failed to scp pcap files from %s" % src.name )
             # Grab logs
-            main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True, includeCaseDesc=False )
+            useStern = main.params['use_stern'].lower() == "true"
+            main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True,
+                                     includeCaseDesc=False, useStern=useStern,
+                                     startTime=main.eventStart )
             # Grab pcap
             receiverSCP = main.ONOSbench.scp( dst, pcapFileReceiver, main.logdir, direction="from" )
             utilities.assert_equals( expect=main.TRUE, actual=receiverSCP,
@@ -849,6 +853,6 @@
             main.log.warn( "Error opening " + dbFileName + " to write results." )
 
     def cleanup( self, main, headerOrder=None ):
-        run.cleanup( main )
+        run.cleanup( main, copyKarafLog=False )
         main.step( "Writing csv results file for db" )
         self.dbWrite( main, main.TEST + "-dbfile.csv", headerOrder )
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 5f9dd8a..bc4bea6 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -1847,6 +1847,7 @@
         main.step( 'Set logging levels' )
         # Get original values incase we want to reset them
         ctrl = main.Cluster.active(0)
+        ctrl.CLI.clearBuffer( timeout=1 )
         ctrl.CLI.logList()
 
         logging = True
@@ -1870,6 +1871,7 @@
         main.step( 'Reset logging levels' )
         # Get original values incase we want to reset them
         ctrl = main.Cluster.active(0)
+        ctrl.CLI.clearBuffer( timeout=1 )
         currentLevels = ctrl.CLI.logList( saveValues=False )
         origLevels = ctrl.CLI.logLevels
         toBeSet = {}
diff --git a/TestON/tests/dependencies/utils.py b/TestON/tests/dependencies/utils.py
index 8aaca08..e787474 100644
--- a/TestON/tests/dependencies/utils.py
+++ b/TestON/tests/dependencies/utils.py
@@ -54,7 +54,8 @@
                                  onfail="Failed to stopped mininet" )
         return topoResult
 
-    def copyKarafLog( self, copyFileName="", before=False, includeCaseDesc=True ):
+    def copyKarafLog( self, copyFileName="", before=False, includeCaseDesc=True,
+                      useStern=False, startTime=None ):
         """
         Description:
             copy the karaf log and verify it.
@@ -63,6 +64,8 @@
             copyFileName.
         Returns:
         """
+        import datetime
+        import math
         # TODO: Also grab the rotated karaf logs
         main.log.report( "Copy karaf logs" )
         if includeCaseDesc:
@@ -102,12 +105,10 @@
             else:
                 stepResult = main.FALSE and stepResult
         if isKube:
-            # TODO: Look into using Stern, kail, or just use `kubectl logs <pod>`
             # We also need to save the pod name to switch name mapping
             main.ONOSbench.kubectlPodNodes( dstPath=main.logdir + "/podMapping.txt",
                                             kubeconfig=ctrl.k8s.kubeConfig,
                                             namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
-            # TODO Get stratum write logs
             # Save image for pods, based on "describe pods"
             main.ONOSbench.kubectlDescribe( "pods",
                                             main.logdir + "/describePods.txt",
@@ -117,12 +118,28 @@
             pods = main.ONOSbench.kubectlGetPodNames( kubeconfig=ctrl.k8s.kubeConfig,
                                                       namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
 
+            MINUTE = datetime.timedelta( minutes=1 )
             for pod in pods:
                 path = "%s/%s.log" % ( main.logdir, pod )
-                stratumPods = main.ONOSbench.kubectlLogs( pod,
-                                                          path,
-                                                          kubeconfig=ctrl.k8s.kubeConfig,
-                                                          namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
+                if useStern:
+                    if startTime:
+                        now = datetime.datetime.utcnow()
+                        duration = ( now - startTime ) + MINUTE
+                        since = "%ss" % int( math.ceil( duration.total_seconds() ) )
+                    else:
+                        since = "1h"
+                    podResults = main.ONOSbench.sternLogs( pod,
+                                                           path,
+                                                           kubeconfig=ctrl.k8s.kubeConfig,
+                                                           namespace=main.params[ 'kubernetes' ][ 'namespace' ],
+                                                           since=since,
+                                                           wait=10 )
+                else:
+                    podResults = main.ONOSbench.kubectlLogs( pod,
+                                                             path,
+                                                             kubeconfig=ctrl.k8s.kubeConfig,
+                                                             namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
+                stepResult = stepResult and podResults
         utilities.assert_equals( expect=main.TRUE,
                                  actual=stepResult,
                                  onpass="Successfully copied remote ONOS logs",