Fixes for Nightly tests

- Add onos-diagnostics-k8s command
- Don't catch SkipCase Exception
- Minor cleanup for TAP output

Change-Id: I647e6e57bf9351c69d3059a07ef14d826c244ad7
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 72db4c9..a7d4e22 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -357,23 +357,23 @@
             main.TOTAL_TC_NORESULT = main.TOTAL_TC_NORESULT + 1
             main.log.exact( "\n " + "*" * 29 + "\n" + "\n Result: No Assertion Called \n" + "*" * 29 + "\n" )
             line = "Case " + case + ": " + main.CurrentTestCase + " - No Result"
-            main.log.TAP( "ok - %s # TODO No assert called" % line )
+            main.log.TAP( "ok %s # TODO No assert called" % line )
         elif currentResult == 1:
             main.TOTAL_TC_RUN = main.TOTAL_TC_RUN + 1
             main.TOTAL_TC_PASS = main.TOTAL_TC_PASS + 1
             main.log.exact( "\n" + "*" * 29 + "\n Result: Pass \n" + "*" * 29 + "\n" )
             line = "Case " + case + ": " + main.CurrentTestCase + " - PASS"
-            main.log.TAP( "ok - %s" % line )
+            main.log.TAP( "ok %s" % line )
         elif currentResult == 0:
             main.TOTAL_TC_RUN = main.TOTAL_TC_RUN + 1
             main.TOTAL_TC_FAIL = main.TOTAL_TC_FAIL + 1
             main.log.exact( "\n" + "*" * 29 + "\n Result: Failed \n" + "*" * 29 + "\n" )
             line = "Case " + case + ": " + main.CurrentTestCase + " - FAIL"
-            main.log.TAP( "not ok - %s" % line )
+            main.log.TAP( "not ok %s" % line )
         else:
             main.log.error( " Unknown result of case " + case +
                             ". Result was: " + currentResult )
             line = "Case " + case + ": " + main.CurrentTestCase + " - ERROR"
-            main.log.TAP( "not ok - %s" % line )
+            main.log.TAP( "not ok %s" % line )
         main.log.wiki( "<h3>" + line + "</h3>" )
         main.log.summary( line )
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index ff01b00..5653ffe 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -254,6 +254,8 @@
         self.stepResultsList = []
         self.stepName = ""
         self.caseExplanation = ""
+        self.CASERESULT = self.ERROR
+        self.STEPRESULT = self.NORESULT
         result = self.TRUE
 
         # NOTE: number of main.step statements in the
@@ -343,35 +345,26 @@
         """
         Add case results to the TAP results file
         """
-        #self.log.TAP( "<p>" + self.caseExplanation + "</p>" )
         main.log.debug( self.stepCache )
-        subcaseMessage = False
         steps = 0
         stepLines = []
         for line in self.stepCache.splitlines():
-            main.log.debug( line )
             if re.search( "[0-9]\.[0-9]", line ):  # Step
-                if subcaseMessage:  # End of Failure Message Printout
-                    subcaseMessage = False
                 if re.search( " - PASS$", line ):
                     steps += 1
-                    stepLines.append( "    ok -- STEP %s" % line )
+                    stepLines.append( "    ok - STEP %s" % line )
                 elif re.search( " - FAIL$", line ):
                     steps += 1
-                    stepLines.append( "    not ok -- STEP %s" % line )
+                    stepLines.append( "    not ok - STEP %s" % line )
                 elif re.search( " - No Result$", line ):
                     steps += 1
-                    stepLines.append( "    ok -- STEP %s # TODO: No assertion in test step" % line )
+                    stepLines.append( "    ok - STEP %s # TODO: No assertion in test step" % line )
             else:  # Substep
-                if not subcaseMessage:  # Open Failure Message Printout
-                    stepLines.append( "    # %s" % line )
-                    subcaseMessage = True
-                else:  # Add to Failure Message Printout
-                    self.log.TAP( "    # %s" % line )
+                stepLines.append( "    # %s" % line )
         if steps > 0:
             self.log.TAP( "    1..%s" % steps )
-            for line in stepLines:
-                self.log.TAP( line )
+        for line in stepLines:
+            self.log.TAP( line )
 
 
     def organizeResult( self, caseNum, result ):
@@ -467,11 +460,11 @@
         main.log.debug( "StepResultsTAP" )
         for line in self.stepCache.splitlines():
             if re.search( " - PASS$", line ):
-                self.log.TAP( "    ok -- STEP %s" % line )
+                self.log.TAP( "    ok - STEP %s" % line )
             elif re.search( " - FAIL$", line ):
-                self.log.TAP( "    not ok -- STEP %s" % line )
+                self.log.TAP( "    not ok - STEP %s" % line )
             elif re.search( " - No Result$", line ):
-                self.log.TAP( "    ok -- STEP %s # TODO: No assertion in test step" % line )
+                self.log.TAP( "    ok - STEP %s # TODO: No assertion in test step" % line )
             else:  # Should only be on fail message
                 self.log.TAP( "    # %s" % line )
 
@@ -501,7 +494,7 @@
         except Exception:
             self.log.exception( "Error parsing step results" )
 
-    def skipCase( self, result="DEFAULT", msg=None ):
+    def skipCase( self, result="NORESULT", msg=None ):
         """
         Will skip the rest of the code in a test case. The case results will be
         determined as normal based on completed assertions unless the result
@@ -515,7 +508,9 @@
         result = result.upper().strip()
         if result == "PASS":
             self.CASERESULT = self.TRUE
-        elif result == "FAIL":
+        elif result == "NORESULT":
+            self.CASERESULT = self.NORESULT
+        else:
             self.CASERESULT = self.FALSE
         self.onFailMsg = "Skipping the rest of this case. "
         if msg:
diff --git a/TestON/drivers/common/cli/onosclusterdriver.py b/TestON/drivers/common/cli/onosclusterdriver.py
index 8b25d22..b4b6c12 100755
--- a/TestON/drivers/common/cli/onosclusterdriver.py
+++ b/TestON/drivers/common/cli/onosclusterdriver.py
@@ -265,6 +265,10 @@
                                 node.REST.port = localPort
                             elif self.up4Port and port == int( self.up4Port ):
                                 node.p4rtUp4.p4rtPort = localPort
+                        # Set kubeconfig for all components
+                        for shell in [ node.CLI, node.Bench, node.k8s, node.p4rtUp4  ]:
+                            if shell:
+                                shell.setEnv( "KUBECONFIG", value=kubectl.kubeConfig )
                         main.log.info( "Setting up port forward for pod %s: [ %s ]" % ( self.podNames[ index ], portsList ) )
                         pf = kubectl.kubectlPortForward( self.podNames[ index ],
                                                          portsList,
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 4b4329d..b83a962 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -2786,6 +2786,66 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
 
+    def onosDiagnosticsK8s( self, onosNames, dstDir, suffix, timeout=300, profile="TRELLIS_PROFILE",
+                            namespace="tost", karafDir="apache-karaf-4.2.9", diagsCmd="onos-diagnostics-k8s" ):
+        """
+            Run onos-diagnostics-k8s with given ONOS instance IPs and save output to dstDir
+            with suffix specified E.g. onos-diags-suffix.tar.gz
+            required arguments:
+                onosNames - list of ONOS pod names for collecting diags
+                dstDir - diags file will be saved under the directory specified
+                suffix - diags file will be named with the suffix specified
+            returns:
+                main.FALSE if there's an error executing the command, and main.TRUE otherwise
+        """
+        try:
+            self.handle.sendline( "export DIAGS_PROFILE=%s" % profile )
+            self.handle.expect( self.prompt )
+            cmd = "%s -s %s -k %s" % ( diagsCmd, namespace, karafDir )
+            assert isinstance( onosNames, list )
+            for pod in onosNames:
+                cmd += " " + str( pod )
+            self.handle.sendline( cmd )
+            i = 0
+            while i != 2:
+                i = self.handle.expect( [ "Password", ".txt", self.prompt ], timeout=timeout )
+                handle = self.handle.before
+                main.log.debug( "%s: %s" % ( self.name, handle ) )
+                if i == 0:
+                    self.handle.sendline( self.pwd )
+            assert handle is not None, "Error in sendline"
+            assert "The requested URL returned error" not in handle, handle
+            assert "Command not found:" not in handle, handle
+            assert "Exception:" not in handle, handle
+            # Rename and move diags file to dstDir from /tmp
+            if dstDir[ -1: ] != "/":
+                dstDir += "/"
+            self.handle.sendline( "mv /tmp/onos-diags.tar.gz " + str( dstDir ) + "onos-diags" + str( suffix ) + ".tar.gz" )
+            self.handle.expect( self.prompt )
+            handle = self.handle.before
+            main.log.debug( "%s: %s" % ( self.name, handle ) )
+            assert handle is not None, "Error in sendline"
+            assert "No such file or directory" not in handle, handle
+            return main.TRUE
+        except AssertionError:
+            main.log.exception( "{} Error in onos-diagnostics-k8s output:".format( self.name ) )
+            return main.FALSE
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.FALSE
+        except pexpect.TIMEOUT:
+            main.log.exception( self.name + ": TIMEOUT exception found in onosDiagnostics-k8s" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            self.exitFromCmd( self.prompt, 100 )
+            return main.FALSE
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
     def onosPower( self, onosIP, toggle, userName=None ):
         """
             Run onos-power script to tell the cell warden to simulate a power faulure
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index 4213e0d..e65be4e 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -102,6 +102,8 @@
             else:
                 # Run the test with physical devices
                 run.connectToPhysicalNetwork( main, hostDiscovery=False )  # We don't want to do host discovery in the pod
+        except SkipCase:
+            raise
         except Exception as e:
             main.log.exception( "Error in setupTest" )
             main.skipCase( result="FAIL", msg=e )
@@ -297,6 +299,8 @@
             main.funcs.startTshark( main, src, pingDesc=pingDesc, direction="Sender",
                                     srcIp=srcIp, dstIp=dstIp, protocolStr=protocolStr )
 
+        except SkipCase:
+            raise
         except Exception as e:
             main.log.exception( "Error in setupFlow" )
             main.skipCase( result="FAIL", msg=e )
@@ -435,6 +439,8 @@
             # Timestamp used for EVENT START
             main.eventStart = datetime.datetime.utcnow()
             return switchComponent, srcList, dstList
+        except SkipCase:
+            raise
         except Exception as e:
             main.log.exception( "Error in startCapturing" )
             main.skipCase( result="FAIL", msg=e )
@@ -547,6 +553,8 @@
                 colName = "%s" % "%s-dropped-packets" % receiverResultDesc
                 main.downtimeResults[ colName[:63] ] = dropped
 
+        except SkipCase:
+            raise
         except Exception as e:
             main.log.exception( "Error in stopFlow" )
             main.skipCase( result="FAIL", msg=e )
@@ -637,6 +645,8 @@
                                          onpass="Saved write-req file from %s" % switch,
                                          onfail="Failed to cp write-req file from %s" % switch )
             """
+        except SkipCase:
+            raise
         except Exception:
             main.log.exception( "Error in stopCapturing" )
 
@@ -674,6 +684,8 @@
             # We need another way of uploading, this doesn't have guaranteed order and # of fields
             # main.downtimeResults.update( componentBreakdownDict )
             main.log.debug( json.dumps( main.downtimeResults, sort_keys=True, indent=4 ) )
+        except SkipCase:
+            raise
         except Exception:
             main.log.exception( "Error while breaking down logs" )
 
@@ -767,6 +779,8 @@
             # This is not currently working, disabling for now
             # main.funcs.analyzeLogs( shortDesc, 'portstate_down', main.eventStart, main.eventStop, main.logdir )
             return device, port
+        except SkipCase:
+            raise
         except Exception:
             main.log.exception( "Error in linkDown" )
 
@@ -828,6 +842,8 @@
             # Break down logs
             # This is not currently working, disabling for now
             # main.funcs.analyzeLogs( shortDesc, 'portstate_up', main.eventStart, main.eventStop, main.logdir )
+        except SkipCase:
+            raise
         except Exception:
             main.log.exception( "Error in linkUp" )
 
@@ -941,6 +957,8 @@
             # This is not currently working, disabling for now
             # main.funcs.analyzeLogs( shortDescRecovery, 'start_onl', main.eventStart, main.eventStop, main.logdir )
             # Check the switch is back in ONOS
+        except SkipCase:
+            raise
         except Exception:
             main.log.exception( "Error in onlReboot" )
 
@@ -1062,6 +1080,8 @@
             # Break down logs
             # This is not currently working, disabling for now
             # main.funcs.analyzeLogs( shortDescRecovery, 'powerup_switch', main.eventStart, main.eventStop, main.logdir )
+        except SkipCase:
+            raise
         except Exception:
             main.log.exception( "Error in killSwitchAgent" )
 
@@ -1069,6 +1089,8 @@
     def onosDown():
         try:
             pass
+        except SkipCase:
+            raise
         except Exception:
             main.log.exception( "Error in onosDown" )
 
@@ -1145,6 +1167,8 @@
 
             # TODO What to return? List of touples? [(duration, dropped Packets),...] ?
             return output
+        except SkipCase:
+            raise
         except Exception as e:
             main.log.exception( "Error in analyzeIperfPcap" )
 
@@ -1163,6 +1187,8 @@
                 component.handle.send( "\x03" )  # CTRL-C
                 component.handle.expect( component.prompt, timeout=10 )
                 main.log.debug( component.handle.before + str( component.handle.after ) )
+            except SkipCase:
+                raise
             except Exception:
                 main.log.exception( "Error in onosDown" )
                 return -1
@@ -1199,6 +1225,8 @@
                     return delta
             main.log.error( "No Packets found" )
             return 0
+        except SkipCase:
+            raise
         except Exception:
             main.log.exception( "Error in analyzePcap" )
 
@@ -1244,6 +1272,8 @@
                 main.log.debug( port )
                 targetsStats[ device ] = deltaStats
             return targetsStats
+        except SkipCase:
+            raise
         except Exception as e:
             main.log.exception( "Error in portstatsDelta" )
             main.log.debug( "Initial: %s\nUpdated: %s\n" % (initialStats, updatedStats) )
@@ -1284,6 +1314,8 @@
                 main.log.warn( "Delta not above threshold of %s" % threshold )
                 return None, None
             return retDevice, retPort
+        except SkipCase:
+            raise
         except Exception as e:
             main.log.exception( "Error in findPortWithTraffic" )
             main.log.debug( "Initial: %s\nUpdated: %s\n" % ( initialStats, updatedStats ) )
@@ -1320,6 +1352,8 @@
                     switchComponent = main.Network.switches[ switch ]
             main.log.debug( switchComponent )
             return switchComponent
+        except SkipCase:
+            raise
         except Exception as e:
             main.log.exception( "Error in findSwitchWithTraffic" )
             main.skipCase( result="FAIL", msg=e )
@@ -1335,6 +1369,8 @@
                 if switchComponent.shortName in device[ 'id' ]:
                     return device[ 'available' ]
             return False
+        except SkipCase:
+            raise
         except Exception as e:
             main.log.exception( "Error in switchIsConnected" )
             main.skipCase( result="FAIL", msg=e )
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 23b1984..93af0ee 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -463,8 +463,14 @@
         suffix: suffix string of the file name. E.g. onos-diags-case1.tar.gz
         """
         main.log.info( "Collecting onos-diags..." )
+        podNames = []
         for ctrl in main.Cluster.runningNodes:
-            main.ONOSbench.onosDiagnostics( [ctrl.ipAddress], main.logdir,"-CASE%d" % main.CurrentTestCaseNumber, onosPortnumber=ctrl.REST.port )
+            if ctrl.k8s:
+                podNames.append( ctrl.k8s.podName )
+            else:
+                main.ONOSbench.onosDiagnostics( [ctrl.ipAddress], main.logdir, "-CASE%d" % main.CurrentTestCaseNumber, onosPortnumber=ctrl.REST.port )
+        if podNames:
+            main.ONOSbench.onosDiagnosticsK8s( podNames, main.logdir, "-CASE%d" % main.CurrentTestCaseNumber )
 
     @staticmethod
     def config( main, cfgName ):