[ONOS-7937] Automatically check SCPF test results
Change-Id: Ifec2e7965fbc43ab9bb65338810ca62d7b120739
(cherry picked from commit aa7bc72e78ba2ba5656939232751b1501cc56ceb)
diff --git a/TestON/JenkinsFile/JenkinsfileTrigger b/TestON/JenkinsFile/JenkinsfileTrigger
index 9d4e420..056945d 100644
--- a/TestON/JenkinsFile/JenkinsfileTrigger
+++ b/TestON/JenkinsFile/JenkinsfileTrigger
@@ -30,12 +30,15 @@
// read the passed parameters from the Jenkins.
machines = params.machines
manually_run = params.manual_run
+# FIXME: for now we disable any other slack notifications excpet alarms
+/*
if ( !manually_run ){
slackSend( color: '#03CD9F',
message: ":sparkles:" * 16 + "\n" +
"Starting tests on : " + now.toString() +
"\n" + ":sparkles:" * 16 )
}
+*/
// store the list of the machines into the dictionary.
machineList = machines.tokenize( "\n;, " )
diff --git a/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy b/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
index b3948b4..d65737e 100644
--- a/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
+++ b/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
@@ -239,7 +239,7 @@
sudo rm ${WORKSPACE}/*Wiki.txt
sudo rm ${WORKSPACE}/*Summary.txt
sudo rm ${WORKSPACE}/*Result.txt
- sudo rm ${WORKSPACE}/*Alarm.txt
+ sudo rm ${WORKSPACE}/*Alarm.txt || true
sudo rm ${WORKSPACE}/*.csv
#copy files to workspace
cd `ls -t ~/OnosSystemTest/TestON/logs/*/ | head -1 | sed 's/://'`
diff --git a/TestON/tests/SCPF/SCPFcbench/SCPFcbench.params b/TestON/tests/SCPF/SCPFcbench/SCPFcbench.params
index 80b2382..8feaeaf 100644
--- a/TestON/tests/SCPF/SCPFcbench/SCPFcbench.params
+++ b/TestON/tests/SCPF/SCPFcbench/SCPFcbench.params
@@ -11,8 +11,8 @@
<availableNodes>1</availableNodes>
<ENV>
- <cellName>Cbenchcell</cellName>
- <cellApps>drivers,openflow-base,fwd</cellApps>
+ <cellName>Cbenchcell</cellName>
+ <cellApps>drivers,openflow-base,fwd</cellApps>
</ENV>
<TEST>
@@ -44,7 +44,8 @@
<user>sdn</user>
</CBENCH>
- <JSON>
- </JSON>
+ <ALARM>
+ <minTp>1000000</minTp>
+ </ALARM>
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFcbench/SCPFcbench.py b/TestON/tests/SCPF/SCPFcbench/SCPFcbench.py
index 4a09129..d74ff8f 100644
--- a/TestON/tests/SCPF/SCPFcbench/SCPFcbench.py
+++ b/TestON/tests/SCPF/SCPFcbench/SCPFcbench.py
@@ -117,6 +117,12 @@
main.log.info( "Average: \t\t\t" + avg )
main.log.info( "Standard Deviation: \t" + stdev )
+ # Check cbench test results
+ threshold = float( main.params[ 'ALARM' ][ 'minTp' ].split( ',' )[ main.cycle - 1 ] )
+ if float( avg ) < threshold:
+ main.log.alarm( "{}-node: {}/s < {}/s".format( main.Cluster.numCtrls,
+ avg, threshold ) )
+
try:
dbFileName = "/tmp/CbenchDB"
dbfile = open( dbFileName, "w+" )
diff --git a/TestON/tests/SCPF/SCPFflowTp1g/SCPFflowTp1g.params b/TestON/tests/SCPF/SCPFflowTp1g/SCPFflowTp1g.params
index b374227..f414319 100644
--- a/TestON/tests/SCPF/SCPFflowTp1g/SCPFflowTp1g.params
+++ b/TestON/tests/SCPF/SCPFflowTp1g/SCPFflowTp1g.params
@@ -1,16 +1,14 @@
<PARAMS>
- <testcases>0,1,2,1,2,1,2,1,2,1,2,1,2,1,2</testcases>
+ <testcases>0,1,2,1,2,1,2,1,2,1,2</testcases>
<GRAPH>
<nodeCluster>BM</nodeCluster>
<builds>20</builds>
</GRAPH>
- <SCALE>1,3,3,5,5,7,7</SCALE>
- <max>7</max>
-
-
+ <SCALE>1,3,3,5,5</SCALE>
+ <max>5</max>
<ENV>
<cellName>flowTP</cellName>
@@ -21,9 +19,9 @@
<enableFlowRuleStoreBackup>true</enableFlowRuleStoreBackup>
<warmUp>5</warmUp> #number of runs to warm up the system
<sampleSize>20</sampleSize> #number of runs to take data from
- <neighbors>0,0,a,0,a,0,a</neighbors> #list of number of neighbors, a = all
+ <neighbors>0,0,a,0,a</neighbors> #list of number of neighbors, a = all
<flows>122500</flows>
- <flowObj>False</flowObj> #flow objectives will be tested
+ <flowObj>False</flowObj> #flow objectives will be tested
<flowObjectives>10000</flowObjectives>
<flowObjType>filter</flowObjType> #flow objective type i.e. either forward
<switches>35</switches>
@@ -34,12 +32,13 @@
<testCMD3> -t </testCMD3> #flow objective type
<flowResultFile>/tmp/flowTP1gDB</flowResultFile>
<flowObjResultFile>/tmp/flowTP1gDBFlowObj</flowObjResultFile>
-
</TEST>
+
<CFG>
<flowRule>org.onosproject.store.flow.impl.DistributedFlowRuleStore</flowRule>
<nullProvider>org.onosproject.provider.nil.NullProviders</nullProvider>
</CFG>
+
<GIT>
<pull>False</pull>
<branch>master</branch>
@@ -80,7 +79,9 @@
<ip1>localhost</ip1>
</BENCH>
- <JSON>
- </JSON>
+ <ALARM>
+ <minTp>750,1800,1800,2600,2400</minTp>
+ <minTpFlowObj>60,150,150,250,200</minTpFlowObj>
+ </ALARM>
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFflowTp1g/SCPFflowTp1g.py b/TestON/tests/SCPF/SCPFflowTp1g/SCPFflowTp1g.py
index 8d6a2dc..dc9cb52 100644
--- a/TestON/tests/SCPF/SCPFflowTp1g/SCPFflowTp1g.py
+++ b/TestON/tests/SCPF/SCPFflowTp1g/SCPFflowTp1g.py
@@ -292,8 +292,17 @@
stdTP = numpy.std( tp )
- main.log.info( "Average thoughput: " + str( avgTP ) + " K" + toInstall + "/second" )
- main.log.info( "Standard deviation of throughput: " + str( stdTP ) + " K" + toInstall + "/second" )
+ main.log.info( "Average thoughput: " + str( avgTP ) + " K " + toInstall + "/second" )
+ main.log.info( "Standard deviation of throughput: " + str( stdTP ) + " K " + toInstall + "/second" )
+
+ # Check if throughput result is abnormal
+ if isFlowObj:
+ threshold = float( main.params[ 'ALARM' ][ 'minTpFlowObj' ].split( ',' )[ main.cycle - 1 ] )
+ else:
+ threshold = float( main.params[ 'ALARM' ][ 'minTp' ].split( ',' )[ main.cycle - 1 ] )
+ if avgTP < threshold:
+ main.log.alarm( "{}-node with {} neighbor: {}K/s < {}K/s".format( main.Cluster.numCtrls,
+ neighbors, avgTP, threshold ) )
resultsLog = open( str( resultFile ), "a" )
resultString = ( "'" + main.commit + "'," )
diff --git a/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.params b/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.params
index 17a8bd3..adc0a01 100644
--- a/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.params
+++ b/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.params
@@ -1,12 +1,12 @@
<PARAMS>
- <testcases>0,2,11,20,2,11,20,2,11,20,2,11,20</testcases>
+ <testcases>0,2,11,20,2,11,20,2,11,20</testcases>
<GRAPH>
<nodeCluster>BM</nodeCluster>
<builds>20</builds>
</GRAPH>
- <SCALE>1,3,5,7</SCALE>
+ <SCALE>1,3,5</SCALE>
<ENV>
<cellName>topo_perf_test</cellName>
@@ -54,4 +54,8 @@
<node>baremetal</node>
</DATABASE>
+ <ALARM>
+ <maxLat>5.5,5.5,5.5</maxLat>
+ </ALARM>
+
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.py b/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.py
index 71c4680..a658e23 100644
--- a/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.py
+++ b/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.py
@@ -245,6 +245,12 @@
stdResult = numpy.std( addingHostTime )
main.log.info( "std: {}".format( stdResult ) )
+ # Check test results
+ threshold = float( main.params[ 'ALARM' ][ 'maxLat' ].split( ',' )[ main.cycle - 1 ] )
+ if averageResult > threshold:
+ main.log.alarm( "{}-node: {} ms > {} ms".format( main.Cluster.numCtrls,
+ averageResult, threshold ) )
+
# write to DB file
main.log.info( "Writing results to DS file" )
with open( main.dbFileName, "a" ) as dbFile:
diff --git a/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.params b/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.params
index c295a18..955be53 100644
--- a/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.params
+++ b/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.params
@@ -1,6 +1,6 @@
<PARAMS>
- <testcases>0,1,2,1,2,1,2,1,2,1,2,1,2,1,2</testcases>
+ <testcases>0,1,2,1,2,1,2,1,2,1,2</testcases>
<GRAPH>
<nodeCluster>BM</nodeCluster>
@@ -14,8 +14,8 @@
<cellApps>drivers,null</cellApps>
</ENV>
- <SCALE>1,3,3,5,5,7,7</SCALE>
- <max>7</max>
+ <SCALE>1,3,3,5,5</SCALE>
+ <max>5</max>
<GIT>
<pull>False</pull>
@@ -57,37 +57,44 @@
# timeout for pexpect
<timeout>300</timeout>
</SLEEP>
+
<BENCH>
<user>sdn</user>
<ip1>OCN</ip1>
</BENCH>
<TEST>
- <loadFrom>1,1,1,1,1,1,1</loadFrom> #generate load on server, 1 = generator on
- <numSwitches>10,10,10,10,10,10,10</numSwitches>
<duration>400</duration>
<log_interval>20</log_interval>
<numKeys>40000</numKeys>
<numKeysFlowObj>4000</numKeysFlowObj>
<cyclePeriod>1000</cyclePeriod>
- <neighbors>0,0,a,0,a,0,a</neighbors> #a == all nodes (-1)
+ <neighbors>0,0,a,0,a</neighbors> #a == all nodes (-1)
<skipReleaseResourcesOnWithdrawal>true</skipReleaseResourcesOnWithdrawal>
<flowObj>False</flowObj>
</TEST>
+
<CFG>
<intentManager>org.onosproject.net.intent.impl.IntentManager</intentManager>
<intentConfigRegi>org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator</intentConfigRegi>
<nullProvider>org.onosproject.provider.nil.NullProviders</nullProvider>
<intentPerfInstaller>org.onosproject.intentperf.IntentPerfInstaller</intentPerfInstaller>
</CFG>
+
<DATABASE>
<dbName>/tmp/IntentEventTPDB</dbName>
<dbFlowObj>/tmp/IntentEventTPflowObjDB</dbFlowObj>
</DATABASE>
+
<METRICS>
<intents_rate>intents-events-metrics|grep "Intent Installed Events"|cut -d ' ' -f7</intents_rate>
<intents_withdrawn>intents-events-metrics|grep "Intent Withdrawn Events"|cut -d ' ' -f7</intents_withdrawn>
<intents_failed>intents-events-metrics|grep "Intent Failed Events"|cut -d ' ' -f7</intents_failed>
</METRICS>
+ <ALARM>
+ <minTp>37700,90000,90000,135000,140000</minTp>
+ <minTpFlowObj>1000,3000,3000,5000,5000</minTpFlowObj>
+ </ALARM>
+
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.py b/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.py
index 327a60f..1911942 100644
--- a/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.py
+++ b/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.py
@@ -58,7 +58,6 @@
main.BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
main.BENCHUser = main.params[ 'BENCH' ][ 'user' ]
main.MN1Ip = main.params[ 'MN' ][ 'ip1' ]
- main.numSwitches = ( main.params[ 'TEST' ][ 'numSwitches' ] ).split( "," )
main.skipRelRsrc = main.params[ 'TEST' ][ 'skipReleaseResourcesOnWithdrawal' ]
main.flowObj = main.params[ 'TEST' ][ 'flowObj' ]
main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
@@ -148,10 +147,21 @@
main.log.info( "Stop intent-perf" )
for ctrl in main.Cluster.active():
ctrl.CLI.sendline( "intent-perf-stop" )
+ tp = 0.0
if result:
for ctrl in main.Cluster.active():
main.log.info( "Node {} final Overall Rate: {}".format( ctrl.ipAddress,
result[ ctrl.ipAddress ] ) )
+ tp += float( result[ ctrl.ipAddress ] )
+
+ # Check if throughput result is abnormal
+ if main.flowObj:
+ threshold = float( main.params[ 'ALARM' ][ 'minTpFlowObj' ].split( ',' )[ main.cycle - 1 ] )
+ else:
+ threshold = float( main.params[ 'ALARM' ][ 'minTp' ].split( ',' )[ main.cycle - 1 ] )
+ if tp < threshold:
+ main.log.alarm( "{}-node with {} neighbor: {}/s < {}/s".format( main.Cluster.numCtrls,
+ neighbors, tp, threshold ) )
with open( main.dbFileName, "a" ) as resultDB:
for nodes in range( main.Cluster.numCtrls ):
diff --git a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.params b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.params
index 5ab7277..541d128 100644
--- a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.params
+++ b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.params
@@ -1,14 +1,14 @@
<PARAMS>
- <testcases>0,1,2,1,2,1,2,1,2</testcases>
+ <testcases>0,1,2,1,2,1,2</testcases>
<GRAPH>
<nodeCluster>BM</nodeCluster>
<builds>20</builds>
</GRAPH>
- <SCALE>1,3,5,7</SCALE>
- <max>7</max>
+ <SCALE>1,3,5</SCALE>
+ <max>5</max>
<ENV>
<cellName>IntentInstallWithdrawCell</cellName>
@@ -30,14 +30,17 @@
<pull>False</pull>
<branch>master</branch>
</GIT>
+
<DEPENDENCY>
<FILE1>intentInstallLatFuncs</FILE1>
<PATH>/tests/SCPF/SCPFintentInstallWithdrawLat/dependencies/</PATH>
</DEPENDENCY>
+
<DATABASE>
<dbName>/tmp/IntentInstallWithdrawLatDB</dbName>
<dbFlowObj>/tmp/IntentInstallWithdrawLatDBWFO</dbFlowObj>
</DATABASE>
+
<ATTEMPTS>
<verify>6</verify>
<maxInvalidRun>5</maxInvalidRun>
@@ -52,11 +55,13 @@
# timeout for pexpect
<timeout>300</timeout>
</SLEEP>
+
<CFG>
<intentManager>org.onosproject.net.intent.impl.IntentManager</intentManager>
<intentConfigRegi>org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator</intentConfigRegi>
<nullProvider>org.onosproject.provider.nil.NullProviders</nullProvider>
</CFG>
+
<CTRL>
<USER>sdn</USER>
@@ -91,7 +96,11 @@
<ip1>localhost</ip1>
</BENCH>
- <JSON>
- </JSON>
+ <ALARM>
+ <maxInstallLat>20,22,50,45,50,120,41,50,100</maxInstallLat>
+ <maxWithdrawLat>16,18,40,40,45,90,40,45,80</maxWithdrawLat>
+ <maxInstallLatFlowObj>30,700,7000,60,500,3100,60,200,1800</maxInstallLatFlowObj>
+ <maxWithdrawLatFlowObj>30,900,5500,60,300,3000,55,200,1700</maxWithdrawLatFlowObj>
+ </ALARM>
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py
index 8b798bd..30df5a4 100644
--- a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py
+++ b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py
@@ -276,6 +276,26 @@
main.log.report( "Intent batch: " + str( batchSize ) )
main.log.report( "Install average: {} std: {}".format( installave, installstd ) )
main.log.report( "Withdraw average: {} std: {}".format( withdrawave, withdrawstd ) )
+
+ # Check test results
+ index = len( main.intentsList ) * ( main.cycle - 1 ) + main.intentsList.index( batchSize )
+ # Check installation results
+ if main.flowObj:
+ threshold = float( main.params[ 'ALARM' ][ 'maxInstallLatFlowObj' ].split( ',' )[ index ])
+ else:
+ threshold = float( main.params[ 'ALARM' ][ 'maxInstallLat' ].split( ',' )[ index ])
+ if installave > threshold:
+ main.log.alarm( "{}-node install avg: {} ms > {} ms".format( main.Cluster.numCtrls,
+ installave, threshold ) )
+ # Check withdrawal results
+ if main.flowObj:
+ threshold = float( main.params[ 'ALARM' ][ 'maxWithdrawLatFlowObj' ].split( ',' )[ index ])
+ else:
+ threshold = float( main.params[ 'ALARM' ][ 'maxWithdrawLat' ].split( ',' )[ index ])
+ if withdrawave > threshold:
+ main.log.alarm( "{}-node withdraw avg: {} ms > {} ms".format( main.Cluster.numCtrls,
+ withdrawave, threshold ) )
+
# write result to database file
if not ( numpy.isnan( installave ) or numpy.isnan( installstd ) or
numpy.isnan( withdrawstd ) or numpy.isnan( withdrawave ) ):
diff --git a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params
index d4d1971..2760654 100644
--- a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params
+++ b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params
@@ -1,14 +1,14 @@
<PARAMS>
- <testcases>0,1,2,1,2,1,2,1,2</testcases>
+ <testcases>0,1,2,1,2,1,2</testcases>
<GRAPH>
<nodeCluster>BM</nodeCluster>
<builds>20</builds>
</GRAPH>
- <SCALE>1,3,5,7</SCALE>
- <max>7</max>
+ <SCALE>1,3,5</SCALE>
+ <max>5</max>
<ENV>
<cellName>intentRerouteCell</cellName>
@@ -70,11 +70,13 @@
# timeout for pexpect
<timeout>300</timeout>
</SLEEP>
+
<CFG>
<intentManager>org.onosproject.net.intent.impl.IntentManager</intentManager>
<intentConfigRegi>org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator</intentConfigRegi>
<nullProvider>org.onosproject.provider.nil.NullProviders</nullProvider>
</CFG>
+
<CTRL>
<USER>sdn</USER>
@@ -110,7 +112,9 @@
<ip1>localhost</ip1>
</BENCH>
- <JSON>
- </JSON>
+ <ALARM>
+ <maxLat>20,22,0,35,40,0,30,35,0</maxLat>
+ <maxLatFlowObj>35,1300,50,700,60,600</maxLatFlowObj>
+ </ALARM>
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.py b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.py
index 40790fb..da96551 100644
--- a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.py
+++ b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.py
@@ -18,7 +18,6 @@
You should have received a copy of the GNU General Public License
along with TestON. If not, see <http://www.gnu.org/licenses/>.
"""
-# SCPFintentRerouteLat
"""
SCPFintentRerouteLat
- Test Intent Reroute Latency
@@ -334,6 +333,17 @@
main.log.report( "Global latency std:................" + str( stdGlobalLatency ) )
main.log.report( "________________________________________________________" )
+ # Check test results
+ index = len( main.intentsList ) * ( main.cycle - 1 ) + main.intentsList.index( batchSize )
+ # Check installation results
+ if main.flowObj:
+ threshold = float( main.params[ 'ALARM' ][ 'maxLatFlowObj' ].split( ',' )[ index ])
+ else:
+ threshold = float( main.params[ 'ALARM' ][ 'maxLat' ].split( ',' )[ index ])
+ if aveLocalLatency > threshold:
+ main.log.alarm( "{}-node install avg: {} ms > {} ms".format( main.Cluster.numCtrls,
+ aveLocalLatency, threshold ) )
+
if not ( numpy.isnan( aveLocalLatency ) or numpy.isnan( aveGlobalLatency ) ):
# check if got NaN for result
resultsDB = open( main.dbFileName, "a" )
diff --git a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.params b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.params
index a2444e3..7a08b43 100644
--- a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.params
+++ b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.params
@@ -75,9 +75,9 @@
</JSON>
<ALARM>
- <maxPortUpAve>10,20,21</maxPortUpAve>
+ <maxPortUpAvg>10,20,21</maxPortUpAvg>
<maxPortUpStd>10,10,10</maxPortUpStd>
- <maxPortDownAve>6,6,7</maxPortDownAve>
+ <maxPortDownAvg>6,6,7</maxPortDownAvg>
<maxPortDownStd>2,2,2</maxPortDownStd>
</ALARM>
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.py b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.py
index 9ca7d2c..1a553ea 100644
--- a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.py
+++ b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.py
@@ -231,18 +231,24 @@
"Link to Grapg Std: {}".format( str( resultDict[ d ][ 'node' + str( i ) ][ 'Std' ][ 'LtoG' ] ) ) )
# Check if any result is abnormal
+ # Port-up
result = resultDict[ 'up' ][ 'node' + str( maxDict[ 'up' ][ 'node' ] ) ][ 'Ave' ][ 'EtoE' ]
- if result > float( main.params[ 'ALARM' ][ 'maxPortUpAve' ].split( ',' )[ main.cycle - 1 ] ):
- main.log.alarm( "Average of port up latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ threshold = float( main.params[ 'ALARM' ][ 'maxPortUpAvg' ].split( ',' )[ main.cycle - 1 ] )
+ if result > threshold:
+ main.log.alarm( "{}-node port-up avg: {} ms > {} ms".format( main.Cluster.numCtrls, result, threshold ) )
result = resultDict[ 'up' ][ 'node' + str( maxDict[ 'up' ][ 'node' ] ) ][ 'Std' ][ 'EtoE' ]
- if result > float( main.params[ 'ALARM' ][ 'maxPortUpStd' ].split( ',' )[ main.cycle - 1 ] ):
- main.log.alarm( "Std of port up latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ threshold = float( main.params[ 'ALARM' ][ 'maxPortUpStd' ].split( ',' )[ main.cycle - 1 ] )
+ if result > threshold:
+ main.log.alarm( "{}-node port-up std: {} ms > {} ms".format( main.Cluster.numCtrls, result, threshold ) )
+ # Port-down
result = resultDict[ 'down' ][ 'node' + str( maxDict[ 'down' ][ 'node' ] ) ][ 'Ave' ][ 'EtoE' ]
- if result > float( main.params[ 'ALARM' ][ 'maxPortDownAve' ].split( ',' )[ main.cycle - 1 ] ):
- main.log.alarm( "Average of port down latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ threshold = float( main.params[ 'ALARM' ][ 'maxPortDownAvg' ].split( ',' )[ main.cycle - 1 ] )
+ if result > threshold:
+ main.log.alarm( "{}-node port-down avg: {} ms > {} ms".format( main.Cluster.numCtrls, result, threshold ) )
result = resultDict[ 'down' ][ 'node' + str( maxDict[ 'down' ][ 'node' ] ) ][ 'Std' ][ 'EtoE' ]
- if result > float( main.params[ 'ALARM' ][ 'maxPortDownStd' ].split( ',' )[ main.cycle - 1 ] ):
- main.log.alarm( "Std of port down latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ threshold = float( main.params[ 'ALARM' ][ 'maxPortDownStd' ].split( ',' )[ main.cycle - 1 ] )
+ if result > threshold:
+ main.log.alarm( "{}-node port-down std: {} ms > {} ms".format( main.Cluster.numCtrls, result, threshold ) )
with open( main.dbFileName, "a" ) as dbFile:
# Scale number
diff --git a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
index 39f73bf..02334cf 100755
--- a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
+++ b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
@@ -7,8 +7,9 @@
# 100 - balance master and bring onos node down
# 200 - bring onos node up and balance masters
# 1000 - report logs
+ # 1001 - report alarms
# 1,[2,10,300,11,100,300,11,200,300,11,1000]*3
- <testcases>1,[3,2,10,300,11,1000]*7,3</testcases>
+ <testcases>1,[3,2,10,300,11,1000]*5,3,1001</testcases>
<GRAPH>
<nodeCluster>BM</nodeCluster>
@@ -82,4 +83,8 @@
<host>True</host>
</TOPOLOGY>
+ <ALARM>
+ <minMaxScale>40</minMaxScale>
+ </ALARM>
+
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py
index 5ac38e6..99a246d 100644
--- a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py
+++ b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py
@@ -81,6 +81,7 @@
main.MNLinksTime = int( main.params[ 'TIMEOUT' ][ 'links' ] )
main.getTopoTime = int( main.params[ 'TIMEOUT' ][ 'getTopo' ] )
main.currScale = None
+ main.maxScale = 1
main.threadID = 0
if main.hostDiscover == 'True':
main.hostDiscover = True
@@ -464,6 +465,7 @@
main.log.error( "The results of two capture are different!" )
main.log.debug( "The data is " + str( main.allinfo ) )
if main.writeData != -1:
+ main.maxScale = main.currScale
main.log.info( "Write the date into database" )
# write the date into data base
with open( main.dbFilePath, "a" ) as dbFile:
@@ -490,3 +492,12 @@
"ERROR",
"Except" ],
"s" )
+
+ def CASE1001( self, main ):
+ """
+ Write abnormal test results to alarm log
+ """
+ threshold = int( main.params[ 'ALARM' ][ 'minMaxScale' ] )
+ if int( main.maxScale ) < threshold:
+ main.log.alarm( "Max scale: {}x{} < {}x{}".format( main.maxScale, main.maxScale,
+ threshold, threshold ) )
diff --git a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
index 699baa9..134256b 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
+++ b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
@@ -140,9 +140,9 @@
</JSON>
<ALARM>
- <maxSwitchUpAve>60,60,68</maxSwitchUpAve>
+ <maxSwitchUpAvg>60,60,68</maxSwitchUpAvg>
<maxSwitchUpStd>10,10,10</maxSwitchUpStd>
- <maxSwitchDownAve>5,5,8</maxSwitchDownAve>
+ <maxSwitchDownAvg>5,5,8</maxSwitchDownAvg>
<maxSwitchDownStd>2,2,2</maxSwitchDownStd>
</ALARM>
</PARAMS>
diff --git a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
index f27b4f2..11ca548 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
+++ b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
@@ -250,18 +250,24 @@
main.log.report( "Device to Graph Std: {}".format( str( resultDict[ "down" ][ 'node' + str( i ) ][ 'Std' ][ 'D_G' ] ) ) )
# Check if any result is abnormal
+ # Switch-up
result = resultDict[ 'up' ][ 'node' + str( maxDict[ 'up' ][ 'node' ] ) ][ 'Ave' ][ 'E_E' ]
- if result > float( main.params[ 'ALARM' ][ 'maxSwitchUpAve' ].split( ',' )[ main.cycle - 1 ] ):
- main.log.alarm( "Average of switch up latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ threshold = float( main.params[ 'ALARM' ][ 'maxSwitchUpAvg' ].split( ',' )[ main.cycle - 1 ] )
+ if result > threshold:
+ main.log.alarm( "{}-node switch-up avg: {} ms > {} ms".format( main.Cluster.numCtrls, result, threshold ) )
result = resultDict[ 'up' ][ 'node' + str( maxDict[ 'up' ][ 'node' ] ) ][ 'Std' ][ 'E_E' ]
- if result > float( main.params[ 'ALARM' ][ 'maxSwitchUpStd' ].split( ',' )[ main.cycle - 1 ] ):
- main.log.alarm( "Std of switch up latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ threshold = float( main.params[ 'ALARM' ][ 'maxSwitchUpStd' ].split( ',' )[ main.cycle - 1 ] )
+ if result > threshold:
+ main.log.alarm( "{}-node switch-up std: {} ms > {} ms".format( main.Cluster.numCtrls, result, threshold ) )
+ # Switch-down
result = resultDict[ 'down' ][ 'node' + str( maxDict[ 'down' ][ 'node' ] ) ][ 'Ave' ][ 'E_E' ]
- if result > float( main.params[ 'ALARM' ][ 'maxSwitchDownAve' ].split( ',' )[ main.cycle - 1 ] ):
- main.log.alarm( "Average of switch down latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ threshold = float( main.params[ 'ALARM' ][ 'maxSwitchDownAvg' ].split( ',' )[ main.cycle - 1 ] )
+ if result > threshold:
+ main.log.alarm( "{}-node switch-down avg: {} ms > {} ms".format( main.Cluster.numCtrls, result, threshold ) )
result = resultDict[ 'down' ][ 'node' + str( maxDict[ 'down' ][ 'node' ] ) ][ 'Std' ][ 'E_E' ]
- if result > float( main.params[ 'ALARM' ][ 'maxSwitchDownStd' ].split( ',' )[ main.cycle - 1 ] ):
- main.log.alarm( "Std of switch down latency is {} with cluster size {}".format( result, main.Cluster.numCtrls ) )
+ threshold = float( main.params[ 'ALARM' ][ 'maxSwitchDownStd' ].split( ',' )[ main.cycle - 1 ] )
+ if result > threshold:
+ main.log.alarm( "{}-node switch-down std: {} ms > {} ms".format( main.Cluster.numCtrls, result, threshold ) )
with open( main.dbFileName, "a" ) as dbFile:
# Scale number
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index 6384c3a..a3f0e2b 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -440,6 +440,7 @@
ctrl.name,
ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
main.log.error( "Failed to start ONOS, stopping test" )
+ main.log.alarm( "Failed to start ONOS: not all nodes are in READY state" )
main.cleanAndExit()
return main.TRUE