TestON/JenkinsFile/CommonJenkinsFile.groovy

#!groovy
// Copyright 2017 Open Networking Foundation (ONF)
//
// Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
// the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
// or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
//
//     TestON is free software: you can redistribute it and/or modify
//     it under the terms of the GNU General Public License as published by
//     the Free Software Foundation, either version 2 of the License, or
//     (at your option) any later version.
//
//     TestON is distributed in the hope that it will be useful,
//     but WITHOUT ANY WARRANTY; without even the implied warranty of
//     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//     GNU General Public License for more details.
//
//     You should have received a copy of the GNU General Public License
//     along with TestON.  If not, see <http://www.gnu.org/licenses/>.

import groovy.time.TimeCategory
import groovy.time.TimeDuration

// read the dependency files
graphs = evaluate readTrusted( 'TestON/JenkinsFile/dependencies/JenkinsGraphs.groovy' )
test_list = evaluate readTrusted( 'TestON/JenkinsFile/dependencies/JenkinsTestONTests.groovy' )
fileRelated = evaluate readTrusted( 'TestON/JenkinsFile/dependencies/JenkinsPathAndFiles.groovy' )
SCPFfuncs = evaluate readTrusted( 'TestON/JenkinsFile/dependencies/PerformanceFuncs.groovy' )

category = null
prop = null
testsToRun = null
testsToRunStrList = null
branch = null
branchWithPrefix = null
start = null
nodeLabel = null
testStation = null
testsOverride = null
isGraphOnly = false
isSCPF = false
pipelineTimeout = null

testsFromList = [:]
graphPaths = [:]
pipeline = [:]

main()

def main(){
    pipelineTimeout = params.TimeOut.toInteger() // integer minutes until the entire pipeline times out. Usually passed from upstream master-trigger job.
    timeout( time: pipelineTimeout, unit: "MINUTES" ){
        init()
        runTests()
        generateGraphs()
        sendToSlack()
    }
}

def init(){
    fileRelated.init()
    test_list.init()
    readParams()

    if ( branch == "manually" ){
        echo '''Warning: entered branch was: "manually". Defaulting to master branch.'''
        branch = "master"
        branchWithPrefix = test_list.addPrefixToBranch( branch )
    }

    if ( category == "SCPF" ){
        isSCPF = true
        SCPFfuncs.init()
        graphs.initialize( SCPFfuncs );
        prop = getProperties()
        isOldFlow = ( prop[ "isOldFlow" ] == "true" )
        SCPFfuncs.oldFlowRuleCheck( isOldFlow, prop[ "ONOSBranch" ] )
    } else {
        isSCPF = false
        graphs.initialize()
        prop = getProperties()
    }

    // get the list of the tests from category
    testsFromList = test_list.getTestsFromCategory( category )

    initGraphPaths()
    tokenizeTokens = "\n;, "

    if ( testsOverride == "" || testsOverride == null ){
        testsToRunStrList = prop[ "Tests" ].tokenize( tokenizeTokens )
    } else {
        testsToRunStrList = testsOverride.tokenize( tokenizeTokens )
    }
    testsToRun = test_list.getTestsFromStringList( testsToRunStrList )
}

def readParams(){
    category = params.Category       // "FUNC", "HA", "USECASE", etc.
    branch = params.Branch           // "1.15", "2.1", "master", etc.
    branchWithPrefix = test_list.addPrefixToBranch( branch )
    testStation = params.TestStation // "TestStation-BMs", etc.
    nodeLabel = params.NodeLabel     // "BM", "VM", "Fabric-1.x", etc.
    testsOverride = params.TestsOverride // "FUNCflow, FUNCintent, [...]", overrides property file
    isGraphOnly = params.OnlyRefreshGraphs // true or false
}

def getProperties(){
    // get the properties of the test by reading the TestONOS.property

    filePath = '''/var/jenkins/TestONOS-''' + category + '''-''' + branchWithPrefix + '''.property'''

    node( testStation ) {
        return readProperties( file: filePath )
    }
}

def getCurrentTime(){
    // get time of the PST zone.

    TimeZone.setDefault( TimeZone.getTimeZone( 'PST' ) )
    return new Date()
}

def initGraphPaths(){
    graphPaths.put( "trendIndividual", fileRelated.rScriptPaths[ "scripts" ][ "trendIndividual" ] )
    if ( category == "SR" || category == "SR-StratumBMv2" ){
        graphPaths.put( "saveDirectory", fileRelated.workspaces[ "base" ] + "postjob-" + ( testStation - "TestStation-" - "s" ) + "/" )
    } else if ( category == "SRHA" ) {
        graphPaths.put( "saveDirectory", fileRelated.workspaces[ "Fabric" ] )
    } else if ( category == "SCPF" || category == "USECASE" ){
        graphPaths.put( "saveDirectory", fileRelated.workspaces[ "BM" ] )
    } else {
        graphPaths.put( "saveDirectory", fileRelated.workspaces[ "VM" ] )
    }
}

def runTests(){
    // run the test sequentially and save the function into the dictionary.
    for ( String test : testsFromList.keySet() ){
        toBeRun = testsToRun.keySet().contains( test )
        stepName = ( toBeRun ? "" : "Not " ) + "Running $test"
        // pureTestName is what is passed to the cli, here we check to  see if there are any params to pass as well
        if ( test.contains( "WithFlowObj" ) ){
            pureTestName = test - "WithFlowObj"
        } else if ( toBeRun && testsToRun[ test ].keySet().contains( "test" )  ){
            pureTestName = testsToRun[ test ][ "test" ]
        } else {
            pureTestName = test
        }
        pipeline[ stepName ] = runTest( test,
                                        toBeRun,
                                        prop,
                                        pureTestName,
                                        isGraphOnly,
                                        testsToRun,
                                        graphPaths[ "trendIndividual" ],
                                        graphPaths[ "saveDirectory" ] )
    }

    // get the start time of the test.
    start = getCurrentTime()

    // run the tests sequentially.
    for ( test in pipeline.keySet() ){
        pipeline[ test ].call()
    }
}

def initTest(){
    return '''#!/bin/bash -l
        set -i # interactive
        set +e
        shopt -s expand_aliases # expand alias in non-interactive mode
        export PYTHONUNBUFFERED=1
        ifconfig
        echo "ONOS Branch is: $ONOSBranch"
        echo "TestON Branch is: $TestONBranch"
        echo "Test date: "
        date
        cd ~
        export PATH=$PATH:onos/tools/test/bin
        timeout 240 stc shutdown | head -100
        timeout 240 stc teardown | head -100
        timeout 240 stc shutdown | head -100
        cd ~/OnosSystemTest/TestON/bin
        git log | head
        ./cleanup.sh -f || true
        '''
}

def configureJavaVersion(){
    java_1_8_branches = [ "1.15" ]
    return '''#!/bin/bash -l
        ''' + ( java_1_8_branches.contains( branch ) ? '''SET_JAVA_VER=java-8-oracle''' : '''SET_JAVA_VER=java-1.11.0-openjdk-amd64''' ) +
        '''
           CELL_COUNT=$( env | egrep "OC[1-9]+" | wc -l )
           for i in $(seq 1 $CELL_COUNT)
           do
               CELL_TO_SET=$( env | egrep "OC$i" | cut -c5- )
               echo "Setting java to $SET_JAVA_VER on $CELL_TO_SET"
               eval ssh $CELL_TO_SET 'sudo update-java-alternatives -s $SET_JAVA_VER'
           done
        '''
}

def runTestCli_py( testName, pureTestName, testCategory ){
    // Bash script that will run the test.
    // testName : name of the test
    // testCategory : (SR,FUNC ... )
    flowObjFlag = false

    if ( isSCPF && testName.contains( "WithFlowObj" ) ){
        flowObjFlag = true
    }

    command = '''cd ~/OnosSystemTest/TestON/bin
                 ./cli.py run ''' + pureTestName + ''' --params''' + ( flowObjFlag ? '''TEST/flowObj=True ''' : ''' ''' ) + '''GRAPH/nodeCluster=''' + graphs.getPostjobType( nodeLabel ) + ''' '''
    echo command

    return command


}

def concludeRunTest(){
    return '''cd ~/OnosSystemTest/TestON/bin
              ./cleanup.sh -f || true
              # cleanup config changes
              cd ~/onos/tools/package/config
              git clean -df'''
}

def copyLogs(){
    // bash script to copy the logs and other necessary element for SR tests.

    result = ""
    if ( category == "SR" || category == "SR-StratumBMv2" ){
        result = '''
      sudo rm /var/jenkins/workspace/SR-log-${WikiPrefix}/*
      sudo cp *karaf.log.* /var/jenkins/workspace/SR-log-${WikiPrefix}/
      sudo cp *Flows* /var/jenkins/workspace/SR-log-${WikiPrefix}/
      sudo cp *Groups* /var/jenkins/workspace/SR-log-${WikiPrefix}/
      sudo cp *.tar.gz /var/jenkins/workspace/SR-log-${WikiPrefix}/
      sudo cp t3-* /var/jenkins/workspace/SR-log-${WikiPrefix}/
      '''
    }
    return result
}

def cleanAndCopyFiles( testName ){
    // clean up some files that were in the folder and copy the new files from the log
    // testName : name of the test

    return '''#!/bin/bash -i
        set +e
        echo "ONOS Branch is: ${ONOSBranch}"
        echo "TestON Branch is: ${TestONBranch}"
        echo "Job name is: "''' + testName + '''
        echo "Workspace is: ${WORKSPACE}/"
        echo "Wiki page to post is: ${WikiPrefix}-"
        # remove any leftover files from previous tests
        sudo rm ${WORKSPACE}/*Wiki.txt
        sudo rm ${WORKSPACE}/*Summary.txt
        sudo rm ${WORKSPACE}/*Result.txt
        sudo rm ${WORKSPACE}/*Alarm.txt || true
        sudo rm ${WORKSPACE}/*.csv
        #copy files to workspace
        cd `ls -t ~/OnosSystemTest/TestON/logs/*/ | head -1 | sed 's/://'`
        ''' + copyLogs() + '''
        sudo cp *.txt ${WORKSPACE}/
        sudo cp *.csv ${WORKSPACE}/
        cd ${WORKSPACE}/
        for i in *.csv
            do mv "$i" "$WikiPrefix"-"$i"
        done
        ls -al
        cd '''
}

def fetchLogs( testName ){
    // fetch the logs of onos from onos nodes to onos System Test logs
    // testName: name of the test

    return '''#!/bin/bash
  set +e
  cd ~/OnosSystemTest/TestON/logs
  echo "TestON test name is: "''' + testName + '''
  TestONlogDir=$(ls -t | grep ${TEST_NAME}_  |head -1)
  echo "########################################################################################"
  echo "#####  copying ONOS logs from all nodes to TestON/logs directory: ${TestONlogDir}"
  echo "########################################################################################"
  cd $TestONlogDir
  if [ $? -eq 1 ]
  then
      echo "Job name does not match any test suite name to move log!"
  else
      pwd
      for i in $OC{1..7}; do onos-fetch-logs $i || echo log does not exist for onos $i; done
      for i in $OC{1..7}; do atomix-fetch-logs $i || echo log does not exist for atomix $i; done
  fi
  cd'''
}

def publishToConfluence( isManualRun, isPostResult, wikiLink, file ){
    // publish HTML script to wiki confluence
    // isManualRun : string "true" "false"
    // isPostResult : string "true" "false"
    // wikiLink : link of the wiki page to publish
    // file : name of the file to be published

    if ( graphs.isPostingResult( isManualRun, isPostResult ) ){
        publishConfluence siteName: 'wiki.onosproject.org', pageName: wikiLink, spaceName: 'ONOS',
                          attachArchivedArtifacts: true, buildIfUnstable: true,
                          editorList: [ confluenceWritePage( confluenceFile( file ) ) ]
    }
}

def postLogs( testName, prefix ){
    // posting logs of the onos jobs specifically SR tests
    // testName : name of the test
    // prefix : branch prefix ( master, 2.1, 1.15 ... )

    resultURL = ""
    if ( category == "SR" || category == "SR-StratumBMv2" ){
        def post = build job: "SR-log-" + prefix, propagate: false
        resultURL = post.getAbsoluteUrl()
    }
    return resultURL
}

def analyzeResult( prop, workSpace, pureTestName, testName, resultURL, wikiLink, isSCPF ){
    // analyzing the result of the test and send to slack if any abnormal result is logged.
    // prop : property dictionary
    // workSpace : workSpace where the result file is saved
    // pureTestName : TestON name of the test
    // testName : Jenkins name of the test. Example: SCPFflowTPFobj
    // resultURL : url for the logs for SR tests. Will not be posted if it is empty
    // wikiLink : link of the wiki page where the result was posted
    // isSCPF : Check if it is SCPF. If so, it won't post the wiki link.

    node( testStation ) {
        def alarmFile = workSpace + "/" + pureTestName + "Alarm.txt"
        if ( fileExists( alarmFile ) ) {
            def alarmContents = readFile( alarmFile )
            slackSend( channel: "#jenkins-related",
                       color: "FF0000",
                       message: "[" + prop[ "ONOSBranch" ] + "]" + testName + " : triggered alarms:\n" +
                                alarmContents + "\n" +
                                "[TestON log] : \n" +
                                "https://jenkins.onosproject.org/blue/organizations/jenkins/${ env.JOB_NAME }/detail/${ env.JOB_NAME }/${ env.BUILD_NUMBER }/pipeline" +
                                ( isSCPF ? "" : ( "\n[Result on Wiki] : \n" +
                                                  "https://wiki.onosproject.org/display/ONOS/" +
                                                  wikiLink.replaceAll( "\\s", "+" ) ) ) +
                                ( resultURL != "" ? ( "\n[Karaf log] : \n" +
                                                      resultURL + "artifact/" ) : "" ),
                       teamDomain: 'onosproject' )
            print "Abnormal test result."
            throw new Exception( "Abnormal test result." )
        }
        else {
            print "Test results are OK."
        }
    }
}

def runTest( testName, toBeRun, prop, pureTestName, graphOnly, testCategory, graph_generator_file,
             graph_saved_directory ){
    // run the test on the machine that contains all the steps : init and run test, copy files, publish result ...
    // testName : name of the test in Jenkins
    // toBeRun : boolean value whether the test will be run or not. If not, it won't be run but shows up with empty
    //           result on pipeline view
    // prop : dictionary property on the machine
    // pureTestName : Pure name of the test. ( ex. pureTestName of SCPFflowTpFobj will be SCPFflowTp )
    // graphOnly : check if it is generating graph job. If so, it will only generate the generating graph part
    // testCategory : Map for the test suit ( SCPF, SR, FUNC, ... ) which contains information about the tests
    // graph_generator_file : Rscript file with the full path.
    // graph_saved_directory : where the generated graph will be saved to.

    return {
        catchError {
            stage( testName ) {
                if ( toBeRun ){
                    def workSpace = "/var/jenkins/workspace/" + testName
                    def fileContents = ""
                    node( testStation ) {
                        withEnv( [ 'ONOSBranch=' + prop[ "ONOSBranch" ],
                                   'ONOSJAVAOPTS=' + prop[ "ONOSJAVAOPTS" ],
                                   'TestONBranch=' + prop[ "TestONBranch" ],
                                   'ONOSTag=' + prop[ "ONOSTag" ],
                                   'WikiPrefix=' + prop[ "WikiPrefix" ],
                                   'WORKSPACE=' + workSpace ] ) {
                            if ( !graphOnly ){
                                if ( isSCPF ){
                                    // Remove the old database file
                                    sh SCPFfuncs.cleanupDatabaseFile( testName )
                                }
                                sh script: configureJavaVersion(), label: "Configure Java Version"
                                sh script: initTest(), label: "Test Initialization: stc shutdown; stc teardown; ./cleanup.sh"
                                catchError{
                                    sh script: runTestCli_py( testName, pureTestName, testCategory ), label: ( "Run Test: ./cli.py run " + testName )
                                }
                                catchError{
                                    sh script: concludeRunTest(), label: "Conclude Running Test: ./cleanup.sh; git clean -df"
                                }
                                catchError{
                                    // For the Wiki page
                                    sh script: cleanAndCopyFiles( pureTestName ), label: "Clean and Copy Files"
                                }
                            }
                            graphs.databaseAndGraph( prop, testName, pureTestName, graphOnly,
                                                    graph_generator_file, graph_saved_directory )
                            if ( !graphOnly ){
                                sh script: fetchLogs( pureTestName ), label: "Fetch Logs"
                                if ( !isSCPF ){
                                    publishToConfluence( prop[ "manualRun" ], prop[ "postResult" ],
                                                         prop[ "WikiPrefix" ] + "-" + testCategory[ testName ][ 'wikiName' ],
                                                         workSpace + "/" + testCategory[ testName ][ 'wikiFile' ] )
                                }
                            }
                        }
                    }
                    graphs.postResult( prop, graphOnly, nodeLabel )
                    if ( !graphOnly ){
                        def resultURL = postLogs( testName, prop[ "WikiPrefix" ] )
                        analyzeResult( prop, workSpace, pureTestName, testName, resultURL,
                                       isSCPF ? "" : testCategory[ testName ][ 'wikiName' ],
                                       isSCPF )
                    }
                } else {
                    echo testName + " is not being run today. Leaving the rest of stage contents blank."
                }
            }
        }
    }
}

def generateGraphs(){
    if ( category != "SCPF" ){
        // generate the overall graph of the non SCPF tests.
        graphs.generateOverallGraph( prop, testsToRun, graphPaths[ "saveDirectory" ], nodeLabel, category )
    }
}

def sendToSlack(){
    // send the result of the test to the slack when it is not manually running.
    // start : start time of the test
    // isManualRun : string that is whether "false" or "true"
    // branch : branch of the onos.

    try {
        if ( prop[ "manualRun" ] == "false" ){
            end = getCurrentTime()
            TimeDuration duration = TimeCategory.minus( end, start )
            // FIXME: for now we disable notifications of normal test results
            /*
            slackSend( color: "#5816EE",
                       message: category + "-" + prop[ "WikiPrefix" ] + " tests ended at: " + end.toString() +
                                "\nTime took : " + duration )
            */
        }
    }
    catch ( all ){
    }
}