blob: 1977fcc05b32faf2a6832113bae272d84c2f807d [file] [log] [blame]
#!groovy
// Copyright 2019 Open Networking Foundation (ONF)
//
// Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
// the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
// or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
//
// TestON is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 2 of the License, or
// (at your option) any later version.
//
// TestON is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with TestON. If not, see <http://www.gnu.org/licenses/>.
// This is the Jenkins script for master-trigger
import groovy.time.TimeCategory
import groovy.time.TimeDuration
// set the functions of the dependencies.
graphs = evaluate readTrusted( 'TestON/JenkinsFile/dependencies/JenkinsGraphs.groovy' )
fileRelated = evaluate readTrusted( 'TestON/JenkinsFile/dependencies/JenkinsPathAndFiles.groovy' )
test_list = evaluate readTrusted( 'TestON/JenkinsFile/dependencies/JenkinsTestONTests.groovy' )
INITIALIZATION_TIMEOUT_MINUTES = 10 // timeout init() function if it takes too long.
onos_tag = null
manually_run = null
now = null
today = null
onos_branches = null
day = null
post_result = null
branchesParam = null
isFabric = null
testsParam = null
simulateDay = null
pipelineTimeOut = null
dayMap = [:]
fullDayMap = [:]
all_testcases = [:]
runTest = [:]
selectedTests = [:]
graphPaths = [:]
main()
def main() {
timeout( time: INITIALIZATION_TIMEOUT_MINUTES, unit: "MINUTES" ){
init()
}
if ( selectedTests == [:] && manually_run ){
echo "No valid tests provided. Check if the provided test(s) is/are in test.json, and try again.\n\nProvided Tests:\n" + testsParam
throw new Exception( "No valid tests provided. Check if the provided test(s) is/are in test.json, and try again." )
}
timeout( time: pipelineTimeOut, unit: "MINUTES" ){
runTests()
generateGraphs()
}
}
// **************
// Initialization
// **************
// initialize file scope vars
def init(){
// get the name of the job.
jobName = env.JOB_NAME
// set the versions of the onos
fileRelated.init()
test_list.init()
readParams()
// list of the tests to be run will be saved in each choices.
day = ""
initDates()
onos_branches = getONOSBranches()
selectedTests = getONOSTests()
initGraphPaths()
echo "selectedTests: " + selectedTests
echo "onos_branches: " + onos_branches
}
def readParams(){
// get post result from the params for manually run.
post_result = params.PostResult
manually_run = params.manual_run
onos_tag = params.ONOSTag
branchesParam = params.branches
isOldFlow = true // hardcoding to true since we are always using oldFlow.
testsParam = params.Tests
isFabric = params.isFabric
simulateDay = params.simulate_day
pipelineTimeOut = params.TimeOut.toInteger()
}
// Set tests based on day of week
def initDates(){
echo "-> initDates()"
now = getCurrentTime()
dayMap = [ ( Calendar.MONDAY ) : "mon",
( Calendar.TUESDAY ) : "tue",
( Calendar.WEDNESDAY ) : "wed",
( Calendar.THURSDAY ) : "thu",
( Calendar.FRIDAY ) : "fri",
( Calendar.SATURDAY ) : "sat",
( Calendar.SUNDAY ) : "sun" ]
fullDayMap = [ ( Calendar.MONDAY ) : "Monday",
( Calendar.TUESDAY ) : "Tuesday",
( Calendar.WEDNESDAY ) : "Wednesday",
( Calendar.THURSDAY ) : "Thursday",
( Calendar.FRIDAY ) : "Friday",
( Calendar.SATURDAY ) : "Saturday",
( Calendar.SUNDAY ) : "Sunday" ]
if ( simulateDay == "" ){
today = now[ Calendar.DAY_OF_WEEK ]
day = dayMap[ today ]
print now.toString()
} else {
day = simulateDay
}
}
def getCurrentTime(){
// get time of the PST zone.
TimeZone.setDefault( TimeZone.getTimeZone( 'PST' ) )
return new Date()
}
// gets ONOS branches from params or string parameter
def getONOSBranches(){
echo "-> getONOSBranches()"
if ( manually_run ){
return branchesParam.tokenize( "\n;, " )
} else {
return test_list.getBranchesFromDay( day )
}
}
def getONOSTests(){
echo "-> getONOSTests()"
if ( manually_run ){
return test_list.getTestsFromStringList( testsParam.tokenize( "\n;, " ) )
} else {
return test_list.getTestsFromDay( day )
}
}
// init paths for the files and directories.
def initGraphPaths(){
graphPaths.put( "histogramMultiple", fileRelated.rScriptPaths[ "scripts" ][ "histogramMultiple" ] )
graphPaths.put( "pieMultiple", fileRelated.rScriptPaths[ "scripts" ][ "pieMultiple" ] )
graphPaths.put( "saveDirectory", fileRelated.workspaces[ "VM" ] )
}
// **********************
// Determine Tests to Run
// **********************
def printTestsToRun( runList ){
if ( manually_run ){
println "Tests to be run manually:"
} else {
if ( isFabric ){
postToSlackSR()
}
if ( today == Calendar.MONDAY ){
postToSlackTestsToRun()
}
println "Defaulting to " + day + " tests:"
}
for ( list in runList ){
echo "" + list
}
}
def postToSlackSR(){
// If it is automated running, it will post the beginning message to the channel.
slackSend( channel: 'sr-failures', color: '#03CD9F',
message: ":sparkles:" * 16 + "\n" +
"Starting tests on : " + now.toString() +
"\n" + ":sparkles:" * 16 )
}
def postToSlackTestsToRun(){
slackSend( color: '#FFD988',
message: "Tests to be run this weekdays : \n" +
printDaysForTest() )
}
def printDaysForTest(){
// Print the days for what test has.
AllTheTests = test_list.getAllTests()
result = ""
for ( String test in AllTheTests.keySet() ){
result += test + ": ["
test_schedule = AllTheTests[ test ][ "schedules" ]
for ( String sch_dict in test_schedule ){
for ( String day in test_list.convertScheduleKeyToDays( sch_dict[ "branch" ] ) ){
result += day + " "
}
}
result += "]\n"
}
return result
}
// *********
// Run Tests
// *********
def tagCheck( onos_tag, onos_branch ){
// check the tag for onos if it is not empty
result = "git checkout "
if ( onos_tag == "" ){
//create new local branch
result += onos_branch
}
else {
//checkout the tag
result += onos_tag
}
return result
}
def preSetup( onos_branch, test_branch, onos_tag, isManual, category ){
// pre setup part which will clean up and checkout to corresponding branch.
result = ""
if ( !isManual ){
if ( category == "MO" ){
result = '''echo -e "\n##### Set onos-config branch"
cd ~/onos-test/
git checkout HEAD~1 # Make sure you aren't on a branch
git branch | grep -v "detached from" | xargs git branch -d # delete all local branches merged with remote
git branch -D ''' + test_branch + ''' # just in case there are local changes. This will normally result in a branch not found error
git reset --hard origin/''' + test_branch + ''' # force local index to match remote branch
git clean -df # clean any local files
git fetch --all # update all caches from remotes
git reset --hard origin/''' + onos_branch + ''' # force local index to match remote branch
git clean -df
git branch
git log -1 --decorate
'''
} else if ( category == "SR-Tofino" ){
result = '''echo -e "\n##### Set TestON Branch #####"
echo "TestON Branch is set on: ''' + test_branch + '''"
cd ~/OnosSystemTest/
git checkout HEAD~1 # Make sure you aren't pn a branch
git branch | grep -v "detached from" | xargs git branch -d # delete all local branches merged with remote
git branch -D ''' + test_branch + ''' # just in case there are local changes. This will normally result in a branch not found error
git clean -df # clean any local files
git fetch --all # update all caches from remotes
git reset --hard origin/''' + test_branch + ''' # force local index to match remote branch
git clean -df # clean any local files
git checkout ''' + test_branch + ''' #create new local branch
git branch
git log -1 --decorate
'''
} else {
result = '''echo -e "\n##### Set TestON Branch #####"
echo "TestON Branch is set on: ''' + test_branch + '''"
cd ~/OnosSystemTest/
git checkout HEAD~1 # Make sure you aren't pn a branch
git branch | grep -v "detached from" | xargs git branch -d # delete all local branches merged with remote
git branch -D ''' + test_branch + ''' # just in case there are local changes. This will normally result in a branch not found error
git clean -df # clean any local files
git fetch --all # update all caches from remotes
git reset --hard origin/''' + test_branch + ''' # force local index to match remote branch
git clean -df # clean any local files
git checkout ''' + test_branch + ''' #create new local branch
git branch
git log -1 --decorate
echo -e "\n##### Set ONOS Branch #####"
echo "ONOS Branch is set on: ''' + onos_branch + '''"
echo -e "\n #### check karaf version ######"
env |grep karaf
cd ~/onos
git checkout HEAD~1 # Make sure you aren't pn a branch
git branch | grep -v "detached from" | xargs git branch -d # delete all local branches merged with remote
git branch -D ''' + onos_branch + ''' # just incase there are local changes. This will normally result in a branch not found error
git clean -df # clean any local files
git fetch --all # update all caches from remotes
git reset --hard origin/''' + onos_branch + ''' # force local index to match remote branch
git clean -df # clean any local files
rm -rf buck-out
rm -rf bazel-*
''' + tagCheck( onos_tag, onos_branch ) + '''
git branch
git log -1 --decorate
echo -e "\n##### set jvm heap size to 8G #####"
echo ${ONOSJAVAOPTS}
inserted_line="export JAVA_OPTS=\"\${ONOSJAVAOPTS}\""
sed -i "s/bash/bash\\n$inserted_line/" ~/onos/tools/package/bin/onos-service
echo "##### Check onos-service setting..... #####"
cat ~/onos/tools/package/bin/onos-service
export JAVA_HOME=/usr/lib/jvm/java-8-oracle'''
}
} else {
if ( category == "MO" ) {
result = '''echo "Since this is a manual run, we'll use the current onos-test branch:"
echo "onos-test branch:"
cd ~/onos-test
git branch
'''
} else {
result = '''echo "Since this is a manual run, we'll use the current ONOS and TestON branch:"
echo "ONOS branch:"
cd ~/OnosSystemTest/
git branch
echo "TestON branch:"
cd ~/TestON/
git branch'''
}
}
return result
}
def postSetup( onos_branch, test_branch, onos_tag, isManual ){
// setup that will build ONOS
result = ""
if ( !isManual ){
result = '''echo -e "Installing bazel"
cd ~
rm -rf ci-management
git clone https://gerrit.onosproject.org/ci-management
cd ci-management/jjb/onos/
export GERRIT_BRANCH="''' + onos_branch + '''"
chmod +x install-bazel.sh
./install-bazel.sh
'''
} else {
result = '''echo -e "Since this is a manual run, we will not install Bazel."'''
}
return result
}
def generateKey(){
// generate cluster-key of the onos
try {
sh script: '''
#!/bin/bash -l
set +e
. ~/.bashrc
env
onos-push-bits-through-proxy
onos-gen-cluster-key -f
''', label: "Generate Cluster Key", returnStdout: false
} catch ( all ){
}
}
// Initialize the environment Setup for the onos and OnosSystemTest
def envSetup( onos_branch, test_branch, onos_tag, category, manuallyRun, nodeLabel ){
// to setup the environment using the bash script
stage( "Environment Setup: " + onos_branch + "-" + nodeLabel + "-" + category ) {
// after env: ''' + borrow_mn( category ) + '''
sh script: '''#!/bin/bash -l
set +e
. ~/.bashrc
env
''' + preSetup( onos_branch, test_branch, onos_tag, manuallyRun, category ), label: "Repo Setup", returnStdout: false
if ( category != "MO" ){
//sh script: postSetup( onos_branch, test_branch, onos_tag, manuallyRun ), label: "Install Bazel", returnStdout: false
generateKey()
}
}
}
// export Environment properties.
def exportEnvProperty( onos_branch, test_branch, jobOn, wiki, tests, postResult, manually_run, onosTag, isOldFlow, nodeLabel ){
// export environment properties to the machine.
filePath = "/var/jenkins/TestONOS-" + jobOn + "-" + onos_branch + ".property"
stage( "Property Export: " + onos_branch + "-" + nodeLabel + "-" + jobOn ) {
sh script: '''
echo "ONOSBranch=''' + onos_branch + '''" > ''' + filePath + '''
echo "TestONBranch=''' + test_branch + '''" >> ''' + filePath + '''
echo "ONOSTag=''' + onosTag + '''" >> ''' + filePath + '''
echo "WikiPrefix=''' + wiki + '''" >> ''' + filePath + '''
echo "ONOSJAVAOPTS=''' + env.ONOSJAVAOPTS + '''" >> ''' + filePath + '''
echo "Tests=''' + tests + '''" >> ''' + filePath + '''
echo "postResult=''' + postResult + '''" >> ''' + filePath + '''
echo "manualRun=''' + manually_run + '''" >> ''' + filePath + '''
echo "isOldFlow=''' + isOldFlow + '''" >> ''' + filePath + '''
''', label: "Exporting Property File: " + filePath
}
}
def trigger( branch, tests, nodeLabel, jobOn, manuallyRun, onosTag ){
// triggering function that will setup the environment and determine which pipeline to trigger
println "Job name: " + jobOn + "-pipeline-" + ( manuallyRun ? "manually" : branch )
def wiki = branch
def onos_branch = test_list.addPrefixToBranch( branch )
def test_branch = test_list.addPrefixToBranch( branch )
assignedNode = null
node( label: nodeLabel ) {
envSetup( onos_branch, test_branch, onosTag, jobOn, manuallyRun, nodeLabel )
exportEnvProperty( onos_branch, test_branch, jobOn, wiki, tests, post_result, manuallyRun, onosTag, isOldFlow, nodeLabel )
assignedNode = env.NODE_NAME
}
jobToRun = jobOn + "-pipeline-" + ( manuallyRun ? "manually" : wiki )
build job: jobToRun, propagate: false, parameters: [ [ $class: 'StringParameterValue', name: 'Category', value: jobOn ],
[ $class: 'StringParameterValue', name: 'Branch', value: branch ],
[ $class: 'StringParameterValue', name: 'TestStation', value: assignedNode ],
[ $class: 'StringParameterValue', name: 'NodeLabel', value: nodeLabel ],
[ $class: 'StringParameterValue', name: 'TimeOut', value: pipelineTimeOut.toString() ] ]
}
def trigger_pipeline( branch, tests, nodeLabel, jobOn, manuallyRun, onosTag ){
// nodeLabel : nodeLabel from tests.json
// jobOn : "SCPF" or "USECASE" or "FUNC" or "HA"
// this will return the function by wrapping them up with return{} to prevent them to be
// executed once this function is called to assign to specific variable.
return {
trigger( branch, tests, nodeLabel, jobOn, manuallyRun, onosTag )
}
}
def generateRunList(){
runList = [:]
validSchedules = test_list.getValidSchedules( day )
echo "validSchedules: " + validSchedules
for ( branch in onos_branches ){
runBranch = []
nodeLabels = test_list.getAllNodeLabels( branch, selectedTests )
for ( nodeLabel in nodeLabels ){
selectedNodeLabelTests = test_list.getTestsFromNodeLabel( nodeLabel, branch, selectedTests )
selectedNodeLabelCategories = test_list.getAllTestCategories( selectedNodeLabelTests )
for ( category in selectedNodeLabelCategories ){
selectedNodeLabelCategoryTests = test_list.getTestsFromCategory( category, selectedNodeLabelTests )
filteredList = [:]
for ( key in selectedNodeLabelCategoryTests.keySet() ){
for ( sch in selectedNodeLabelCategoryTests[ key ][ "schedules" ] ){
if ( validSchedules.contains( sch[ "day" ] ) && sch[ "branch" ] == test_list.convertBranchToBranchCode( branch ) || manually_run ){
filteredList.put( key, selectedNodeLabelCategoryTests[ key ] )
break
}
}
}
echo "========================================="
echo "BRANCH: " + branch
echo "CATEGORY: " + category
echo "TESTS: " + filteredList
if ( filteredList != [:] ){
exeTestList = test_list.getTestListAsString( filteredList )
runList.put( branch + "-" + nodeLabel + "-" + category, trigger_pipeline( branch, exeTestList, nodeLabel, category, manually_run, onos_tag ) )
}
}
}
}
return runList
}
def runTests(){
runList = generateRunList()
printTestsToRun( runList )
parallel runList
}
// ***************
// Generate Graphs
// ***************
def generateGraphs(){
// If it is automated running, it will generate the stats graph on VM.
if ( !manually_run ){
for ( String b in onos_branches ){
graphs.generateStatGraph( "TestStation-VMs",
test_list.addPrefixToBranch( b ),
graphPaths[ "histogramMultiple" ],
graphPaths[ "pieMultiple" ],
graphPaths[ "saveDirectory" ],
"VM" )
}
}
}