blob: 1ca7ab299e414105c23c0fb84ddbcf2c8d77b269 [file] [log] [blame]
Devin Lim90803a82017-08-29 13:41:44 -07001#!groovy
Devin Lim61657e42017-10-09 17:24:40 -07002import groovy.time.*
Devin Lim90803a82017-08-29 13:41:44 -07003// This is a Jenkinsfile for a scripted pipeline for the SCPF tests
4// properties([pipelineTriggers([cron('30 19 * * *')])])
5
6// TODO: Exception handling around steps
7
8none = [ "" ]
9batches = [1,100,1000]
10neighbors = ['y', 'n']
Devin Lim61657e42017-10-09 17:24:40 -070011times = [ 'y', 'n' ]
Devin Lim90803a82017-08-29 13:41:44 -070012SCPF = [
Devin Lim0e967162017-11-03 15:59:53 -070013 SCPFcbench: [ flows:false, test:'SCPFcbench', table:'cbench_bm_tests', results:'cbench_bm_results', file:'CbenchDB', rFile:'SCPFcbench.R', extra:none, finalResult:1, graphTitle:['Cbench Test'], dbCols:'avg', dbWhere:'', y_axis:'Throughput (Responses/sec)'],
14 SCPFhostLat: [ flows:false, test:'SCPFhostLat', table:'host_latency_tests', results:'host_latency_results', file:'HostAddLatency', rFile:'SCPFhostLat.R', extra:none,finalResult:1, graphTitle:['Host Latency Test'], dbCols:'avg', dbWhere:'AND scale=5', y_axis:'Latency (ms)'],
15 SCPFportLat: [ flows:false, test:'SCPFportLat', table:'port_latency_details', results:'port_latency_results', file:'/tmp/portEventResultDb', rFile:'SCPFportLat.R', extra:none, finalResult:1, graphTitle:['Port Latency Test - Port Up','Port Latency Test - Port Down'], dbCols:[ 'up_ofp_to_dev_avg,up_dev_to_link_avg,up_link_to_graph_avg', 'down_ofp_to_dev_avg,down_dev_to_link_avg,down_link_to_graph_avg' ], dbWhere:'AND scale=5', y_axis:'Latency (ms)' ],
16 SCPFflowTp1g: [ flows:true, test:'SCPFflowTp1g', table:'flow_tp_tests', results:'flow_tp_results', file:'flowTP1gDB', rFile:'SCPFflowTp1g.R n', extra:neighbors,finalResult:1, graphTitle:['Flow Throughput Test - neighbors=0','Flow Throughput Test - neighbors=4'], dbCols:'avg', dbWhere:[ 'AND scale=5 AND neighbors=0 ','AND scale=5 AND NOT neighbors=0' ], y_axis:'Throughput (,000 Flows/sec)' ],
17 SCPFflowTp1gWithFlowObj: [ flows:true, test:'SCPFflowTp1g --params TEST/flowObj=True', table:'flow_tp_fobj_tests', results:'flow_tp_fobj_results', file:'flowTP1gDBFlowObj', rFile:'SCPFflowTp1g.R y', extra:neighbors, finalResult:0],
18 SCPFscaleTopo: [ flows:false, test:'SCPFscaleTopo', table:'scale_topo_latency_details', results:'scale_topo_latency_results', file:'/tmp/scaleTopoResultDb', rFile:'SCPFscaleTopo.R', extra:none, finalResult:1, graphTitle:['Scale Topology Test'], dbCols:[ 'first_connection_to_last_connection, last_connection_to_last_role_request, last_role_request_to_last_topology' ], dbWhere:'AND scale=20' , y_axis:'Latency (s)'],
19 SCPFswitchLat: [ flows:false, test:'SCPFswitchLat', table:'switch_latency_details', results:'switch_latency_results', file:'/tmp/switchEventResultDb', rFile:'SCPFswitchLat.R', extra:none, finalResult:1, graphTitle:['Switch Latency Test - Switch Up','Switch Latency Test - Switch Down'], dbCols:[ 'tcp_to_feature_reply_avg,feature_reply_to_role_request_avg,role_request_to_role_reply_avg,role_reply_to_device_avg,up_device_to_graph_avg', 'fin_ack_to_ack_avg,ack_to_device_avg,down_device_to_graph_avg' ], dbWhere:'AND scale=5', y_axis:'Latency (ms)' ],
20 SCPFbatchFlowResp: [ flows:true, test:'SCPFbatchFlowResp', table:'batch_flow_tests', results:'batch_flow_results', file:'SCPFbatchFlowRespData', rFile:'SCPFbatchFlowResp.R', extra:none, finalResult:1, graphTitle:['Batch Flow Test - Post', 'Batch Flow Test - Del'], dbCols:[ 'elapsepost, posttoconfrm', 'elapsedel, deltoconfrm' ], dbWhere:'', y_axis:'Latency (ms)'],
21 SCPFintentEventTp: [ flows:true, test:'SCPFintentEventTp', table:'intent_tp_tests', results:'intent_tp_results', file:'IntentEventTPDB', rFile:'SCPFintentEventTp.R n', extra:neighbors, finalResult:1, graphTitle:['Intent Throughput Test - neighbors=0','Intent Throughput Test - neighbors=4'], dbCols:'SUM( avg ) as avg', dbWhere:[ 'AND scale=5 AND neighbors=0 GROUP BY date','AND scale=5 AND NOT neighbors=0 GROUP BY date' ], y_axis:'Throughput (Ops/sec)'],
22 SCPFintentRerouteLat: [ flows:true, test:'SCPFintentRerouteLat', table:'intent_reroute_latency_tests', results:'intent_reroute_latency_results', file:'IntentRerouteLatDB', rFile:'SCPFIntentInstallWithdrawRerouteLat.R n', extra:batches, finalResult:1, graphTitle:['Intent Reroute Test'], dbCols:'avg', dbWhere:'AND scale=5 AND batch_size=100', y_axis:'Latency (ms)'],
23 SCPFscalingMaxIntents: [ flows:true, test:'SCPFscalingMaxIntents', table:'max_intents_tests', results:'max_intents_results', file:'ScalingMaxIntentDB', rFile:'SCPFscalingMaxIntents.R n', extra:times, finalResult:0],
24 SCPFintentEventTpWithFlowObj: [ flows:true, test:'SCPFintentEventTp --params TEST/flowObj=True', table:'intent_tp_fobj_tests', results:'intent_tp_fobj_results', file:'IntentEventTPflowObjDB', rFile:'SCPFintentEventTp.R y', extra:neighbors,finalResult:0],
25 SCPFintentInstallWithdrawLat: [ flows:true, test:'SCPFintentInstallWithdrawLat', table:'intent_latency_tests', results:'intent_latency_results', file:'IntentInstallWithdrawLatDB', rFile:'SCPFIntentInstallWithdrawRerouteLat.R n', extra:batches,finalResult:1, graphTitle:['Intent Installation Test','Intent Withdrawal Test'], dbCols:[ 'install_avg','withdraw_avg' ], dbWhere:'AND scale=5 AND batch_size=100', y_axis:'Latency (ms)'],
26 SCPFintentRerouteLatWithFlowObj: [ flows:true, test:'SCPFintentRerouteLat --params TEST/flowObj=True', table:'intent_reroute_latency_fobj_tests', results:'intent_reroute_latency_fobj_results', file:'IntentRerouteLatDBWithFlowObj', rFile:'SCPFIntentInstallWithdrawRerouteLat.R y', extra:batches, finalResult:0],
27 SCPFscalingMaxIntentsWithFlowObj: [ flows:true, test:'SCPFscalingMaxIntents --params TEST/flowObj=True', table:'max_intents_fobj_tests', results:'max_intents_fobj_results', file:'ScalingMaxIntentDBWFO', rFile:'SCPFscalingMaxIntents.R y', extra:times, finalResult:0],
28 SCPFintentInstallWithdrawLatWithFlowObj: [ flows:true, test:'SCPFintentInstallWithdrawLat --params TEST/flowObj=True', table:'intent_latency_fobj_tests', results:'intent_latency_fobj_results', file:'IntentInstallWithdrawLatDBWFO', rFile:'SCPFIntentInstallWithdrawRerouteLat.R y', extra:batches, finalResult:0],
29 SCPFmastershipFailoverLat: [ flows:false, test:'SCPFmastershipFailoverLat', table:'mastership_failover_tests', results:'mastership_failover_results', file:'mastershipFailoverLatDB', rFile:'SCPFmastershipFailoverLat.R', extra:none, finalResult:1, graphTitle:['Mastership Failover Test'], dbCols:[ 'kill_deact_avg,deact_role_avg' ], dbWhere:'AND scale=5', y_axis:'Latency (ms)' ]
Devin Lim90803a82017-08-29 13:41:44 -070030]
31
32echo("Testcases:")
Jeremy Ronquillo2d2649d2017-09-14 12:53:06 -070033graph_generator_directory = "~/OnosSystemTest/TestON/JenkinsFile/scripts/"
Devin Lim90803a82017-08-29 13:41:44 -070034graph_saved_directory = "/var/jenkins/workspace/Pipeline_postjob_BM/"
35def testsToRun = null
36def prop = null
37node("TestStation-BMs"){
38 prop = readProperties(file:'/var/jenkins/TestONOS.property') // TODO set defaults
39 testsToRun = prop["Tests"].tokenize("\n;, ")
40 for ( String test : testsToRun ) {
41 println test
42 }
43}
Devin Lim0e967162017-11-03 15:59:53 -070044isOldFlow = prop[ "isOldFlow" ]
45oldFlowRuleCheck( isOldFlow )
Devin Lim90803a82017-08-29 13:41:44 -070046def tests = [:]
47for( String test : SCPF.keySet() ){
48 toBeRun = testsToRun.contains( test )
49 def stepName = ( toBeRun ? "" : "Not " ) + "Running $test"
50 tests[stepName] = SCPFTest(test, toBeRun, prop)
51}
52
Devin Lim61657e42017-10-09 17:24:40 -070053def now = new Date()
Devin Lim90803a82017-08-29 13:41:44 -070054// run the tests
55for ( test in tests.keySet() ){
56 tests[test].call()
57}
Devin Lim79af50f2017-10-26 14:26:47 -070058try{
59 if( prop["manualRun"] == "false" ){
60 def end = new Date()
61 TimeDuration duration = TimeCategory.minus( end, now )
62 slackSend( color:"#5816EE", message: "SCPF tests ended at: " + end.toString() + "\nTime took : " + duration )
63 }
Devin Lim61657e42017-10-09 17:24:40 -070064}
Devin Lim79af50f2017-10-26 14:26:47 -070065catch(all){}
66
Devin Lim90803a82017-08-29 13:41:44 -070067// The testName should be the key from the SCPF map
68def SCPFTest( testName, toBeRun, prop ) {
69 return {
Devin Lim79af50f2017-10-26 14:26:47 -070070 catchError{
Devin Lim90803a82017-08-29 13:41:44 -070071 stage(testName) {
72 if ( toBeRun ){
73 workSpace = "/var/jenkins/workspace/"+testName
74 node("TestStation-BMs"){
75 withEnv(['ONOSBranch='+prop["ONOSBranch"],
76 'ONOSJVMHeap='+prop["ONOSJVMHeap"],
77 'TestONBranch='+prop["TestONBranch"],
78 'ONOSTag='+prop["ONOSTag"],
79 'WikiPrefix='+prop["WikiPrefix"],
80 'WORKSPACE='+workSpace]){
81 sh '''#!/bin/bash -l
82 set -i # interactive
Devin Lima0e52eb2017-09-13 18:35:12 -070083 set +e
Devin Lim90803a82017-08-29 13:41:44 -070084 shopt -s expand_aliases # expand alias in non-interactive mode
85 export PYTHONUNBUFFERED=1
86
87 ifconfig
88
89 echo "ONOS Branch is: $ONOSBranch"
90 echo "TestON Branch is: $TestONBranch"
91 echo "Test date: "
92 date
93
94 cd ~
95 export PATH=$PATH:onos/tools/test/bin
96
97 timeout 240 stc shutdown | head -100
98 timeout 240 stc teardown | head -100
99 timeout 240 stc shutdown | head -100
100
101 cd ~/OnosSystemTest/TestON/bin
102 git log |head
103 ./cleanup.sh
104 ''' + "./cli.py run " + SCPF[testName]['test']
105
106 // For moving results
107 sh '''#!/bin/bash -i
Devin Lima0e52eb2017-09-13 18:35:12 -0700108 set +e
Devin Lim90803a82017-08-29 13:41:44 -0700109 # remove any leftover files from previous tests
110 sudo rm ${WORKSPACE}/*Result.txt
111
112 #copy files to workspace
113 cd `ls -t ~/OnosSystemTest/TestON/logs/*/ | head -1 | sed 's/://'`
114 sudo cp *Result.txt ${WORKSPACE}/
115 cd ${WORKSPACE}/
116 ls -al
117 cd '''
118 // Post Results
Devin Lim61657e42017-10-09 17:24:40 -0700119 if( prop["manualRun"] == "false" || prop["postResult"] == "true" ){
120 withCredentials([
121 string(credentialsId: 'db_pass', variable: 'pass'),
122 string(credentialsId: 'db_user', variable: 'user'),
123 string(credentialsId: 'db_host', variable: 'host'),
124 string(credentialsId: 'db_port', variable: 'port')]) {
Devin Lim0e967162017-11-03 15:59:53 -0700125 def database_command = pass + "|psql --host=" + host + " --port=" + port + " --username=" + user + " --password --dbname onostest -c \"INSERT INTO " + SCPF[testName]['table'] + " VALUES('\$DATE','" + SCPF[testName]['results'] + "','\$BUILD_NUMBER', '\$ONOSBranch', " + affectedByOldFlow( isOldFlow, testName ) + "\$line);\""
Devin Lim61657e42017-10-09 17:24:40 -0700126 if (testName == "SCPFscaleTopo" || testName == "SCPFswitchLat" || testName == "SCPFportLat") {
127 database_command = pass + "|psql --host=" + host + " --port=" + port + " --username=" + user + " --password --dbname onostest -c \"INSERT INTO " + SCPF[testName]['table'] + " VALUES('\$DATE','" + SCPF[testName]['results'] + "','\$BUILD_NUMBER', \$line, '\$ONOSBranch');\""
128 }
129 sh '''#!/bin/bash
Devin Lim90803a82017-08-29 13:41:44 -0700130
Devin Lim61657e42017-10-09 17:24:40 -0700131 export DATE=\$(date +%F_%T)
132 cd ~
133 pwd
134 cd /tmp
135 while read line
136 do
Devin Lim90803a82017-08-29 13:41:44 -0700137
Devin Lim61657e42017-10-09 17:24:40 -0700138 echo \$line
139 echo ''' + database_command + '''
Devin Lim90803a82017-08-29 13:41:44 -0700140
Devin Lim61657e42017-10-09 17:24:40 -0700141 done< ''' + SCPF[testName]['file'] + '''
Devin Lim0e967162017-11-03 15:59:53 -0700142 ''' + getGraphCommand( SCPF[testName]['rFile'], SCPF[testName]['extra'], host, port, user, pass, testName, prop["ONOSBranch"], isOldFlow ) + '''
143 ''' + ( SCPF[testName]['finalResult'] ? generateCombinedResultGraph( host,port, user, pass, testName, prop["ONOSBranch"], , isOldFlow ) : "" )
Devin Lim61657e42017-10-09 17:24:40 -0700144 }
Devin Lim90803a82017-08-29 13:41:44 -0700145 }
146 // Fetch Logs
147 sh '''#!/bin/bash
Devin Lima0e52eb2017-09-13 18:35:12 -0700148 set +e
Devin Lim90803a82017-08-29 13:41:44 -0700149 cd ~/OnosSystemTest/TestON/logs
150 echo "Job Name is: ${JOB_NAME}"
151 TestONlogDir=$(ls -t | grep ${TEST_NAME}_ |head -1)
152 echo "########################################################################################"
153 echo "##### copying ONOS logs from all nodes to TestON/logs directory: ${TestONlogDir}"
154 echo "########################################################################################"
155 cd $TestONlogDir
156 if [ $? -eq 1 ]
157 then
158 echo "Job name does not match any test suite name to move log!"
159 else
160 pwd
161 for i in $OC{1..7}; do onos-fetch-logs $i || echo log does not exist; done
162 fi'''
163 }
164 }
Devin Lim61657e42017-10-09 17:24:40 -0700165 if( prop["manualRun"] == "false" || prop["postResult"] == "true" ){
Devin Lim90803a82017-08-29 13:41:44 -0700166 def post = build job: "Pipeline_postjob_BM", propagate: false
167 }
168 node("TestStation-BMs"){
Devin Lim0e967162017-11-03 15:59:53 -0700169 resultContents = readFile workSpace + "/" + testName.replaceAll("WithFlowObj","") + "Result.txt"
Devin Lim79af50f2017-10-26 14:26:47 -0700170 resultContents = resultContents.split("\n")
171 if( resultContents[ 0 ] == "1" ){
172 print "All passed"
173 }else{
174 print "Failed"
175 if( prop["manualRun"] == "false" )
176 slackSend(color:"FF0000", message: "[" + prop["ONOSBranch"] + "]" + testName + " : Failed!\n"
Devin Lim0e967162017-11-03 15:59:53 -0700177 + resultContents[ 1 ] + "\n"
178 + "https://onos-jenkins.onlab.us/blue/organizations/jenkins/${env.JOB_NAME}/detail/${env.JOB_NAME}/${env.BUILD_NUMBER}/pipeline" )
Devin Lim79af50f2017-10-26 14:26:47 -0700179 Failed
180 }
Devin Lim90803a82017-08-29 13:41:44 -0700181 }
182 }
183 }
184 }
185 }
186}
Devin Lim0e967162017-11-03 15:59:53 -0700187def getGraphCommand( rFileName, extras, host, port, user, pass, testName, branchName, isOldFlow ){
Devin Lim90803a82017-08-29 13:41:44 -0700188 result = ""
189 for( extra in extras ){
Devin Lim0e967162017-11-03 15:59:53 -0700190 result += generateGraph( rFileName, " " + extra, host, port, user, pass, testName, branchName, isOldFlow ) + ";"
Devin Lim90803a82017-08-29 13:41:44 -0700191 }
192 return result
193}
Devin Lim0e967162017-11-03 15:59:53 -0700194def generateGraph( rFileName, batch, host, port, user, pass, testName, branchName, isOldFlow ){
195 return "Rscript " + graph_generator_directory + rFileName + " " + host + " " + port + " " + user + " " + pass + " " +
196 testName + " " + branchName + " " + batch + " " + usingOldFlow( isOldFlow, testName ) + graph_saved_directory
Devin Lim90803a82017-08-29 13:41:44 -0700197}
Devin Lim0e967162017-11-03 15:59:53 -0700198def generateCombinedResultGraph( host, port, user, pass, testName, branchName, isOldFlow ){
Devin Lim90803a82017-08-29 13:41:44 -0700199 result = ""
200 for ( int i=0; i< SCPF[testName]['graphTitle'].size(); i++){
Devin Lim0e967162017-11-03 15:59:53 -0700201 result += "Rscript " + graph_generator_directory + "SCPFLineGraph.R " + host + " " + port + " " + user + " " + pass + " \"" + SCPF[testName]['graphTitle'][i] + "\" " +
202 branchName + " " + 50 + " \"SELECT " + checkIfList( testName, 'dbCols', i ) + " FROM " + SCPF[testName]['table'] + " WHERE branch=\'" + branchName + "\' " + sqlOldFlow( isOldFlow, testName ) +
203 checkIfList( testName, 'dbWhere', i ) + " ORDER BY date DESC LIMIT 50\" \"" + SCPF[testName]['y_axis'] + "\" " + hasOldFlow( isOldFlow, testName ) + graph_saved_directory + ";"
Devin Lim90803a82017-08-29 13:41:44 -0700204 }
205 return result
206}
207def checkIfList( testName, forWhich, pos ){
Devin Lima5a41002017-09-19 13:05:04 -0700208 return SCPF[testName][forWhich].getClass().getName() != "java.lang.String" ? SCPF[testName][forWhich][pos] : SCPF[testName][forWhich]
Devin Lim0e967162017-11-03 15:59:53 -0700209}
210def sqlOldFlow( isOldFlow, testName ){
Devin Limba6a4f52017-11-09 09:23:07 -0800211 return SCPF[ testName ][ 'flows' ] ? " AND " + ( isOldFlow == "true" ? "" : "NOT " ) + "is_old_flow " : ""
Devin Lim0e967162017-11-03 15:59:53 -0700212}
213def oldFlowRuleCheck( isOldFlow ){
214 if( isOldFlow == "false" ){
215 SCPF[ 'SCPFflowTp1g' ][ 'test' ] += " --params TEST/flows=6125"
216 SCPF[ 'SCPFbatchFlowResp' ][ 'test' ] += " --params CASE1000/batchSize=100"
217 SCPF[ 'SCPFintentEventTp' ][ 'test' ] += " --params TEST/numKeys=4000"
218 }
219}
220def affectedByOldFlow( isOldFlow, testName ){
221 return SCPF[ testName ][ 'flows' ] ? "" + isOldFlow + ", " : ""
222}
223def usingOldFlow( isOldFlow, testName ){
224 return SCPF[ testName ][ 'flows' ] ? ( isOldFlow == "true" ? "y" : "n" ) + " " : ""
225}
226def hasOldFlow( isOldFlow, testName ){
227 return ( SCPF[ testName ][ 'flows' ] && isOldFlow == "true" ? "y" : "n" ) + " "
Devin Lim90803a82017-08-29 13:41:44 -0700228}