[ONOS-7937] Automatically check switchLat results
Change-Id: If4456d475dea756f84721050321afb07225fc8e1
(cherry picked from commit 28cb85502d78cb24a8cef51ea2ad4cefdd94bc83)
diff --git a/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy b/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
index 0a37eac..01404cf 100644
--- a/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
+++ b/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
@@ -162,9 +162,12 @@
if ( isManualRun == "false" ){
end = getCurrentTime()
TimeDuration duration = TimeCategory.minus( end, start )
+ // FIXME: for now we disable notifications of normal test results
+ /*
slackSend( color: "#5816EE",
message: testType + "-" + branch + " tests ended at: " + end.toString() +
"\nTime took : " + duration )
+ */
}
}
catch ( all ){
@@ -308,11 +311,13 @@
// get name of the slack channel.
// if the test is SR, it will return sr-failures
- return "#" + ( testType == "SR" ? "sr-failures" : "jenkins-related" )
+ // FIXME: For now we move all notifications to #jenkins-related
+ // return "#" + ( testType == "SR" ? "sr-failures" : "jenkins-related" )
+ return "#jenkins-related"
}
def analyzeResult( prop, workSpace, pureTestName, testName, resultURL, wikiLink, isSCPF ){
- // analyzing the result of the test and send to slack if the test was failed.
+ // analyzing the result of the test and send to slack if any abnormal result is logged.
// prop : property dictionary
// workSpace : workSpace where the result file is saved
// pureTestName : TestON name of the test
@@ -322,18 +327,15 @@
// isSCPF : Check if it is SCPF. If so, it won't post the wiki link.
node( testMachine ) {
- def resultContents = readFile( workSpace + "/" + pureTestName + "Result.txt" )
- resultContents = resultContents.split( "\n" )
- if ( resultContents[ 0 ] == "1" ){
- print "All passed"
- }
- else {
- print "Failed"
+ def alarmFile = workSpace + "/" + pureTestName + "Alarm.txt"
+ if ( fileExists( alarmFile ) ) {
+ print "Abnormal test result logged"
+ def alarmContents = readFile( alarmFile )
if ( prop[ "manualRun" ] == "false" ){
slackSend( channel: getSlackChannel(),
color: "FF0000",
- message: "[" + prop[ "ONOSBranch" ] + "]" + testName + " : Failed!\n" +
- resultContents[ 1 ] + "\n" +
+ message: "[" + prop[ "ONOSBranch" ] + "]" + testName + " : triggered alarms:\n" +
+ alarmContents + "\n" +
"[TestON log] : \n" +
"https://jenkins.onosproject.org/blue/organizations/jenkins/${ env.JOB_NAME }/detail/${ env.JOB_NAME }/${ env.BUILD_NUMBER }/pipeline" +
( isSCPF ? "" : ( "\n[Result on Wiki] : \n" +
@@ -345,6 +347,9 @@
}
Failed
}
+ else {
+ print "Test results are normal"
+ }
}
}
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 551bacc..0fdc49a 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -124,6 +124,7 @@
main.SummaryFileName = main.logdir + "/" + main.TEST + "Summary.txt"
main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
main.resultFile = main.logdir + "/" + main.TEST + "Result.txt"
+ main.alarmFileName = main.logdir + "/" + main.TEST + "Alarm.txt"
main.TOTAL_TC_SUCCESS = 0
@@ -214,6 +215,17 @@
main.log.step = step
+ def alarm( msg ):
+ '''
+ Format of the alarm type log defined here.
+ '''
+ main.log._log( 6, msg, "OpenFlowAutoMattion", "OFAutoMation" )
+ main.alarmFile = open( main.alarmFileName, "a+" )
+ main.alarmFile.write( msg + "\n" )
+ main.alarmFile.close()
+
+ main.log.alarm = alarm
+
main.LogFileHandler = logging.FileHandler( main.LogFileName )
self._printHeader( main )
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index 978c206..1edc721 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -803,6 +803,7 @@
self.testCaseResult[ str( self.CurrentTestCaseNumber ) ] = self.FALSE
self.organizeResult( self.CurrentTestCaseNumber, self.FALSE )
self.logger.updateCaseResults( self )
+ self.log.alarm( "Test exited unexpectedly" )
self.cleanup()
self.exit()
diff --git a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
index 8680b5c..699baa9 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
+++ b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
@@ -1,13 +1,13 @@
<PARAMS>
- <testcases>0,1,2,1,2,1,2,1,2</testcases>
+ <testcases>0,1,2,1,2,1,2</testcases>
<GRAPH>
<nodeCluster>BM</nodeCluster>
<builds>20</builds>
</GRAPH>
- <SCALE>1,3,5,7</SCALE>
- <max>7</max>
+ <SCALE>1,3,5</SCALE>
+ <max>5</max>
<ENV>
<cellName>topo_perf_test</cellName>
@@ -61,9 +61,11 @@
<ACK>openflow\ \[ACK\]</ACK>
</down>
</TSHARK>
+
<CFG>
<defaultTopo>org.onosproject.net.topology.impl.DefaultTopologyProvider</defaultTopo>
</CFG>
+
<TEST>
#'on' or 'off' debug mode.
#If on, logging will be more verbose and
@@ -117,7 +119,7 @@
<singleSwThreshold>0,1000</singleSwThreshold>
<tabletFile>tablets_3node.json</tabletFile>
- </TEST>
+ </TEST>
<SLEEP>
<startup>5</startup>
@@ -136,4 +138,11 @@
<linkTimestamp>topologyLinkEventTimestamp</linkTimestamp>
<graphTimestamp>topologyGraphEventTimestamp</graphTimestamp>
</JSON>
+
+ <ALARM>
+ <maxSwitchUpAve>60,60,68</maxSwitchUpAve>
+ <maxSwitchUpStd>10,10,10</maxSwitchUpStd>
+ <maxSwitchDownAve>5,5,8</maxSwitchDownAve>
+ <maxSwitchDownStd>2,2,2</maxSwitchDownStd>
+ </ALARM>
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
index 8048d4f..f27b4f2 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
+++ b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
@@ -230,51 +230,40 @@
main.log.report( "=====node{} Summary:=====".format( str( i ) ) )
main.log.report( "=============Switch up=======" )
-
- main.log.report(
- "End to End average: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Ave' ][ 'E_E' ] ) ) )
- main.log.report(
- "End to End Std: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Std' ][ 'E_E' ] ) ) )
-
- main.log.report(
- "TCP to Feature average: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Ave' ][ 'T_F' ] ) ) )
- main.log.report(
- "TCP to Feature Std: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Std' ][ 'T_F' ] ) ) )
-
- main.log.report(
- "Feature to Device average: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Ave' ][ 'F_D' ] ) ) )
- main.log.report(
- "Feature to Device Std: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Std' ][ 'F_D' ] ) ) )
-
- main.log.report(
- "Device to Graph average: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Ave' ][ 'D_G' ] ) ) )
- main.log.report(
- "Device to Graph Std: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Std' ][ 'D_G' ] ) ) )
+ main.log.report( "End to End average: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Ave' ][ 'E_E' ] ) ) )
+ main.log.report( "End to End Std: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Std' ][ 'E_E' ] ) ) )
+ main.log.report( "TCP to Feature average: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Ave' ][ 'T_F' ] ) ) )
+ main.log.report( "TCP to Feature Std: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Std' ][ 'T_F' ] ) ) )
+ main.log.report( "Feature to Device average: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Ave' ][ 'F_D' ] ) ) )
+ main.log.report( "Feature to Device Std: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Std' ][ 'F_D' ] ) ) )
+ main.log.report( "Device to Graph average: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Ave' ][ 'D_G' ] ) ) )
+ main.log.report( "Device to Graph Std: {}".format( str( resultDict[ "up" ][ 'node' + str( i ) ][ 'Std' ][ 'D_G' ] ) ) )
main.log.report( "=============Switch down=======" )
+ main.log.report( "End to End average: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Ave' ][ 'E_E' ] ) ) )
+ main.log.report( "End to End Std: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Std' ][ 'E_E' ] ) ) )
+ main.log.report( "Fin_ACK to ACK average: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Ave' ][ 'FA_A' ] ) ) )
+ main.log.report( "Fin_ACK to ACK Std: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Std' ][ 'FA_A' ] ) ) )
+ main.log.report( "ACK to Device average: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Ave' ][ 'A_D' ] ) ) )
+ main.log.report( "ACK to Device Std: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Std' ][ 'A_D' ] ) ) )
+ main.log.report( "Device to Graph average: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Ave' ][ 'D_G' ] ) ) )
+ main.log.report( "Device to Graph Std: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Std' ][ 'D_G' ] ) ) )
- main.log.report(
- "End to End average: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Ave' ][ 'E_E' ] ) ) )
- main.log.report(
- "End to End Std: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Std' ][ 'E_E' ] ) ) )
-
- main.log.report(
- "Fin_ACK to ACK average: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Ave' ][ 'FA_A' ] ) ) )
- main.log.report(
- "Fin_ACK to ACK Std: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Std' ][ 'FA_A' ] ) ) )
-
- main.log.report(
- "ACK to Device average: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Ave' ][ 'A_D' ] ) ) )
- main.log.report(
- "ACK to Device Std: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Std' ][ 'A_D' ] ) ) )
-
- main.log.report(
- "Device to Graph average: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Ave' ][ 'D_G' ] ) ) )
- main.log.report(
- "Device to Graph Std: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Std' ][ 'D_G' ] ) ) )
+ # Check if any result is abnormal
+ result = resultDict[ 'up' ][ 'node' + str( maxDict[ 'up' ][ 'node' ] ) ][ 'Ave' ][ 'E_E' ]
+ if result > float( main.params[ 'ALARM' ][ 'maxSwitchUpAve' ].split( ',' )[ main.cycle - 1 ] ):
+ main.log.alarm( "Average of switch up latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ result = resultDict[ 'up' ][ 'node' + str( maxDict[ 'up' ][ 'node' ] ) ][ 'Std' ][ 'E_E' ]
+ if result > float( main.params[ 'ALARM' ][ 'maxSwitchUpStd' ].split( ',' )[ main.cycle - 1 ] ):
+ main.log.alarm( "Std of switch up latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ result = resultDict[ 'down' ][ 'node' + str( maxDict[ 'down' ][ 'node' ] ) ][ 'Ave' ][ 'E_E' ]
+ if result > float( main.params[ 'ALARM' ][ 'maxSwitchDownAve' ].split( ',' )[ main.cycle - 1 ] ):
+ main.log.alarm( "Average of switch down latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ result = resultDict[ 'down' ][ 'node' + str( maxDict[ 'down' ][ 'node' ] ) ][ 'Std' ][ 'E_E' ]
+ if result > float( main.params[ 'ALARM' ][ 'maxSwitchDownStd' ].split( ',' )[ main.cycle - 1 ] ):
+ main.log.alarm( "Std of switch down latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
with open( main.dbFileName, "a" ) as dbFile:
- # TODO: Save STD to Database
# Scale number
temp = str( main.Cluster.numCtrls )
temp += ",'baremetal1'"