Fixes for Nightly tests
- Add onos-diagnostics-k8s command
- Don't catch SkipCase Exception
- Minor cleanup for TAP output
Change-Id: I647e6e57bf9351c69d3059a07ef14d826c244ad7
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 72db4c9..a7d4e22 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -357,23 +357,23 @@
main.TOTAL_TC_NORESULT = main.TOTAL_TC_NORESULT + 1
main.log.exact( "\n " + "*" * 29 + "\n" + "\n Result: No Assertion Called \n" + "*" * 29 + "\n" )
line = "Case " + case + ": " + main.CurrentTestCase + " - No Result"
- main.log.TAP( "ok - %s # TODO No assert called" % line )
+ main.log.TAP( "ok %s # TODO No assert called" % line )
elif currentResult == 1:
main.TOTAL_TC_RUN = main.TOTAL_TC_RUN + 1
main.TOTAL_TC_PASS = main.TOTAL_TC_PASS + 1
main.log.exact( "\n" + "*" * 29 + "\n Result: Pass \n" + "*" * 29 + "\n" )
line = "Case " + case + ": " + main.CurrentTestCase + " - PASS"
- main.log.TAP( "ok - %s" % line )
+ main.log.TAP( "ok %s" % line )
elif currentResult == 0:
main.TOTAL_TC_RUN = main.TOTAL_TC_RUN + 1
main.TOTAL_TC_FAIL = main.TOTAL_TC_FAIL + 1
main.log.exact( "\n" + "*" * 29 + "\n Result: Failed \n" + "*" * 29 + "\n" )
line = "Case " + case + ": " + main.CurrentTestCase + " - FAIL"
- main.log.TAP( "not ok - %s" % line )
+ main.log.TAP( "not ok %s" % line )
else:
main.log.error( " Unknown result of case " + case +
". Result was: " + currentResult )
line = "Case " + case + ": " + main.CurrentTestCase + " - ERROR"
- main.log.TAP( "not ok - %s" % line )
+ main.log.TAP( "not ok %s" % line )
main.log.wiki( "<h3>" + line + "</h3>" )
main.log.summary( line )
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index ff01b00..5653ffe 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -254,6 +254,8 @@
self.stepResultsList = []
self.stepName = ""
self.caseExplanation = ""
+ self.CASERESULT = self.ERROR
+ self.STEPRESULT = self.NORESULT
result = self.TRUE
# NOTE: number of main.step statements in the
@@ -343,35 +345,26 @@
"""
Add case results to the TAP results file
"""
- #self.log.TAP( "<p>" + self.caseExplanation + "</p>" )
main.log.debug( self.stepCache )
- subcaseMessage = False
steps = 0
stepLines = []
for line in self.stepCache.splitlines():
- main.log.debug( line )
if re.search( "[0-9]\.[0-9]", line ): # Step
- if subcaseMessage: # End of Failure Message Printout
- subcaseMessage = False
if re.search( " - PASS$", line ):
steps += 1
- stepLines.append( " ok -- STEP %s" % line )
+ stepLines.append( " ok - STEP %s" % line )
elif re.search( " - FAIL$", line ):
steps += 1
- stepLines.append( " not ok -- STEP %s" % line )
+ stepLines.append( " not ok - STEP %s" % line )
elif re.search( " - No Result$", line ):
steps += 1
- stepLines.append( " ok -- STEP %s # TODO: No assertion in test step" % line )
+ stepLines.append( " ok - STEP %s # TODO: No assertion in test step" % line )
else: # Substep
- if not subcaseMessage: # Open Failure Message Printout
- stepLines.append( " # %s" % line )
- subcaseMessage = True
- else: # Add to Failure Message Printout
- self.log.TAP( " # %s" % line )
+ stepLines.append( " # %s" % line )
if steps > 0:
self.log.TAP( " 1..%s" % steps )
- for line in stepLines:
- self.log.TAP( line )
+ for line in stepLines:
+ self.log.TAP( line )
def organizeResult( self, caseNum, result ):
@@ -467,11 +460,11 @@
main.log.debug( "StepResultsTAP" )
for line in self.stepCache.splitlines():
if re.search( " - PASS$", line ):
- self.log.TAP( " ok -- STEP %s" % line )
+ self.log.TAP( " ok - STEP %s" % line )
elif re.search( " - FAIL$", line ):
- self.log.TAP( " not ok -- STEP %s" % line )
+ self.log.TAP( " not ok - STEP %s" % line )
elif re.search( " - No Result$", line ):
- self.log.TAP( " ok -- STEP %s # TODO: No assertion in test step" % line )
+ self.log.TAP( " ok - STEP %s # TODO: No assertion in test step" % line )
else: # Should only be on fail message
self.log.TAP( " # %s" % line )
@@ -501,7 +494,7 @@
except Exception:
self.log.exception( "Error parsing step results" )
- def skipCase( self, result="DEFAULT", msg=None ):
+ def skipCase( self, result="NORESULT", msg=None ):
"""
Will skip the rest of the code in a test case. The case results will be
determined as normal based on completed assertions unless the result
@@ -515,7 +508,9 @@
result = result.upper().strip()
if result == "PASS":
self.CASERESULT = self.TRUE
- elif result == "FAIL":
+ elif result == "NORESULT":
+ self.CASERESULT = self.NORESULT
+ else:
self.CASERESULT = self.FALSE
self.onFailMsg = "Skipping the rest of this case. "
if msg:
diff --git a/TestON/drivers/common/cli/onosclusterdriver.py b/TestON/drivers/common/cli/onosclusterdriver.py
index 8b25d22..b4b6c12 100755
--- a/TestON/drivers/common/cli/onosclusterdriver.py
+++ b/TestON/drivers/common/cli/onosclusterdriver.py
@@ -265,6 +265,10 @@
node.REST.port = localPort
elif self.up4Port and port == int( self.up4Port ):
node.p4rtUp4.p4rtPort = localPort
+ # Set kubeconfig for all components
+ for shell in [ node.CLI, node.Bench, node.k8s, node.p4rtUp4 ]:
+ if shell:
+ shell.setEnv( "KUBECONFIG", value=kubectl.kubeConfig )
main.log.info( "Setting up port forward for pod %s: [ %s ]" % ( self.podNames[ index ], portsList ) )
pf = kubectl.kubectlPortForward( self.podNames[ index ],
portsList,
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 4b4329d..b83a962 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -2786,6 +2786,66 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+ def onosDiagnosticsK8s( self, onosNames, dstDir, suffix, timeout=300, profile="TRELLIS_PROFILE",
+ namespace="tost", karafDir="apache-karaf-4.2.9", diagsCmd="onos-diagnostics-k8s" ):
+ """
+ Run onos-diagnostics-k8s with given ONOS instance IPs and save output to dstDir
+ with suffix specified E.g. onos-diags-suffix.tar.gz
+ required arguments:
+ onosNames - list of ONOS pod names for collecting diags
+ dstDir - diags file will be saved under the directory specified
+ suffix - diags file will be named with the suffix specified
+ returns:
+ main.FALSE if there's an error executing the command, and main.TRUE otherwise
+ """
+ try:
+ self.handle.sendline( "export DIAGS_PROFILE=%s" % profile )
+ self.handle.expect( self.prompt )
+ cmd = "%s -s %s -k %s" % ( diagsCmd, namespace, karafDir )
+ assert isinstance( onosNames, list )
+ for pod in onosNames:
+ cmd += " " + str( pod )
+ self.handle.sendline( cmd )
+ i = 0
+ while i != 2:
+ i = self.handle.expect( [ "Password", ".txt", self.prompt ], timeout=timeout )
+ handle = self.handle.before
+ main.log.debug( "%s: %s" % ( self.name, handle ) )
+ if i == 0:
+ self.handle.sendline( self.pwd )
+ assert handle is not None, "Error in sendline"
+ assert "The requested URL returned error" not in handle, handle
+ assert "Command not found:" not in handle, handle
+ assert "Exception:" not in handle, handle
+ # Rename and move diags file to dstDir from /tmp
+ if dstDir[ -1: ] != "/":
+ dstDir += "/"
+ self.handle.sendline( "mv /tmp/onos-diags.tar.gz " + str( dstDir ) + "onos-diags" + str( suffix ) + ".tar.gz" )
+ self.handle.expect( self.prompt )
+ handle = self.handle.before
+ main.log.debug( "%s: %s" % ( self.name, handle ) )
+ assert handle is not None, "Error in sendline"
+ assert "No such file or directory" not in handle, handle
+ return main.TRUE
+ except AssertionError:
+ main.log.exception( "{} Error in onos-diagnostics-k8s output:".format( self.name ) )
+ return main.FALSE
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return main.FALSE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found in onosDiagnostics-k8s" )
+ main.log.error( self.name + ": " + self.handle.before )
+ self.exitFromCmd( self.prompt, 100 )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
def onosPower( self, onosIP, toggle, userName=None ):
"""
Run onos-power script to tell the cell warden to simulate a power faulure
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
index 4213e0d..e65be4e 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -102,6 +102,8 @@
else:
# Run the test with physical devices
run.connectToPhysicalNetwork( main, hostDiscovery=False ) # We don't want to do host discovery in the pod
+ except SkipCase:
+ raise
except Exception as e:
main.log.exception( "Error in setupTest" )
main.skipCase( result="FAIL", msg=e )
@@ -297,6 +299,8 @@
main.funcs.startTshark( main, src, pingDesc=pingDesc, direction="Sender",
srcIp=srcIp, dstIp=dstIp, protocolStr=protocolStr )
+ except SkipCase:
+ raise
except Exception as e:
main.log.exception( "Error in setupFlow" )
main.skipCase( result="FAIL", msg=e )
@@ -435,6 +439,8 @@
# Timestamp used for EVENT START
main.eventStart = datetime.datetime.utcnow()
return switchComponent, srcList, dstList
+ except SkipCase:
+ raise
except Exception as e:
main.log.exception( "Error in startCapturing" )
main.skipCase( result="FAIL", msg=e )
@@ -547,6 +553,8 @@
colName = "%s" % "%s-dropped-packets" % receiverResultDesc
main.downtimeResults[ colName[:63] ] = dropped
+ except SkipCase:
+ raise
except Exception as e:
main.log.exception( "Error in stopFlow" )
main.skipCase( result="FAIL", msg=e )
@@ -637,6 +645,8 @@
onpass="Saved write-req file from %s" % switch,
onfail="Failed to cp write-req file from %s" % switch )
"""
+ except SkipCase:
+ raise
except Exception:
main.log.exception( "Error in stopCapturing" )
@@ -674,6 +684,8 @@
# We need another way of uploading, this doesn't have guaranteed order and # of fields
# main.downtimeResults.update( componentBreakdownDict )
main.log.debug( json.dumps( main.downtimeResults, sort_keys=True, indent=4 ) )
+ except SkipCase:
+ raise
except Exception:
main.log.exception( "Error while breaking down logs" )
@@ -767,6 +779,8 @@
# This is not currently working, disabling for now
# main.funcs.analyzeLogs( shortDesc, 'portstate_down', main.eventStart, main.eventStop, main.logdir )
return device, port
+ except SkipCase:
+ raise
except Exception:
main.log.exception( "Error in linkDown" )
@@ -828,6 +842,8 @@
# Break down logs
# This is not currently working, disabling for now
# main.funcs.analyzeLogs( shortDesc, 'portstate_up', main.eventStart, main.eventStop, main.logdir )
+ except SkipCase:
+ raise
except Exception:
main.log.exception( "Error in linkUp" )
@@ -941,6 +957,8 @@
# This is not currently working, disabling for now
# main.funcs.analyzeLogs( shortDescRecovery, 'start_onl', main.eventStart, main.eventStop, main.logdir )
# Check the switch is back in ONOS
+ except SkipCase:
+ raise
except Exception:
main.log.exception( "Error in onlReboot" )
@@ -1062,6 +1080,8 @@
# Break down logs
# This is not currently working, disabling for now
# main.funcs.analyzeLogs( shortDescRecovery, 'powerup_switch', main.eventStart, main.eventStop, main.logdir )
+ except SkipCase:
+ raise
except Exception:
main.log.exception( "Error in killSwitchAgent" )
@@ -1069,6 +1089,8 @@
def onosDown():
try:
pass
+ except SkipCase:
+ raise
except Exception:
main.log.exception( "Error in onosDown" )
@@ -1145,6 +1167,8 @@
# TODO What to return? List of touples? [(duration, dropped Packets),...] ?
return output
+ except SkipCase:
+ raise
except Exception as e:
main.log.exception( "Error in analyzeIperfPcap" )
@@ -1163,6 +1187,8 @@
component.handle.send( "\x03" ) # CTRL-C
component.handle.expect( component.prompt, timeout=10 )
main.log.debug( component.handle.before + str( component.handle.after ) )
+ except SkipCase:
+ raise
except Exception:
main.log.exception( "Error in onosDown" )
return -1
@@ -1199,6 +1225,8 @@
return delta
main.log.error( "No Packets found" )
return 0
+ except SkipCase:
+ raise
except Exception:
main.log.exception( "Error in analyzePcap" )
@@ -1244,6 +1272,8 @@
main.log.debug( port )
targetsStats[ device ] = deltaStats
return targetsStats
+ except SkipCase:
+ raise
except Exception as e:
main.log.exception( "Error in portstatsDelta" )
main.log.debug( "Initial: %s\nUpdated: %s\n" % (initialStats, updatedStats) )
@@ -1284,6 +1314,8 @@
main.log.warn( "Delta not above threshold of %s" % threshold )
return None, None
return retDevice, retPort
+ except SkipCase:
+ raise
except Exception as e:
main.log.exception( "Error in findPortWithTraffic" )
main.log.debug( "Initial: %s\nUpdated: %s\n" % ( initialStats, updatedStats ) )
@@ -1320,6 +1352,8 @@
switchComponent = main.Network.switches[ switch ]
main.log.debug( switchComponent )
return switchComponent
+ except SkipCase:
+ raise
except Exception as e:
main.log.exception( "Error in findSwitchWithTraffic" )
main.skipCase( result="FAIL", msg=e )
@@ -1335,6 +1369,8 @@
if switchComponent.shortName in device[ 'id' ]:
return device[ 'available' ]
return False
+ except SkipCase:
+ raise
except Exception as e:
main.log.exception( "Error in switchIsConnected" )
main.skipCase( result="FAIL", msg=e )
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 23b1984..93af0ee 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -463,8 +463,14 @@
suffix: suffix string of the file name. E.g. onos-diags-case1.tar.gz
"""
main.log.info( "Collecting onos-diags..." )
+ podNames = []
for ctrl in main.Cluster.runningNodes:
- main.ONOSbench.onosDiagnostics( [ctrl.ipAddress], main.logdir,"-CASE%d" % main.CurrentTestCaseNumber, onosPortnumber=ctrl.REST.port )
+ if ctrl.k8s:
+ podNames.append( ctrl.k8s.podName )
+ else:
+ main.ONOSbench.onosDiagnostics( [ctrl.ipAddress], main.logdir, "-CASE%d" % main.CurrentTestCaseNumber, onosPortnumber=ctrl.REST.port )
+ if podNames:
+ main.ONOSbench.onosDiagnosticsK8s( podNames, main.logdir, "-CASE%d" % main.CurrentTestCaseNumber )
@staticmethod
def config( main, cfgName ):