TestON/JenkinsFile/JenkinsfileTrigger

#!groovy

funcs = evaluate readTrusted( 'TestON/JenkinsFile/JenkinsCommonFuncs.groovy' )
funcs.initializeTrend( "VM" );
previous_version = "1.12"
before_previous_version = "1.11"
AllTheTests=
[
    "FUNC":[
            "FUNCipv6Intent" : [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "FUNCoptical" :    [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "FUNCflow" :       [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "FUNCnetCfg":      [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "FUNCovsdbtest" :  [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "FUNCnetconf" :    [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "FUNCgroup" :      [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "FUNCformCluster" :[ "basic":false, "extra_A":false, "extra_B":false, "new_Test":true, "day":"" ],
            "FUNCintent" :     [ "basic":false, "extra_A":true, "extra_B":false, "new_Test":false, "day":"" ],
            "FUNCintentRest" : [ "basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":"" ],
    ],
    "HA":[
            "HAsanity" :                [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "HAclusterRestart" :        [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "HAsingleInstanceRestart" : [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "HAupgrade" :               [ "basic":false, "extra_A":false, "extra_B":false, "new_Test":true, "day":"" ],
            "HAupgradeRollback" :       [ "basic":false, "extra_A":false, "extra_B":false, "new_Test":true, "day":"" ],
            "HAstopNodes" :             [ "basic":false, "extra_A":true, "extra_B":false, "new_Test":false, "day":"" ],
            "HAfullNetPartition" :      [ "basic":false, "extra_A":true, "extra_B":false, "new_Test":false, "day":"" ],
            "HAswapNodes" :             [ "basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":"" ],
            "HAscaling" :               [ "basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":"" ],
            "HAkillNodes" :             [ "basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":"" ]
    ],
    "SCPF":[
            "SCPFswitchLat":                           [ "basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFcbench":                              [ "basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFportLat":                             [ "basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFflowTp1g":                            [ "basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFintentEventTp":                       [ "basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFhostLat":                             [ "basic":false, "extra_A":true, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFbatchFlowResp":                       [ "basic":false, "extra_A":true, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFintentRerouteLat":                    [ "basic":false, "extra_A":true, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFintentInstallWithdrawLat":            [ "basic":false, "extra_A":true, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFflowTp1gWithFlowObj":                 [ "basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFintentEventTpWithFlowObj":            [ "basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFintentRerouteLatWithFlowObj":         [ "basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFscalingMaxIntentsWithFlowObj":        [ "basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFintentInstallWithdrawLatWithFlowObj": [ "basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFscaleTopo":                           [ "basic":false, "extra_A":false, "extra_B":false, "extra_C":true, "extra_D":false, "new_Test":false, day:"" ],
            "SCPFscalingMaxIntents":                   [ "basic":false, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":true, "new_Test":false, day:"" ],
            "SCPFmastershipFailoverLat":               [ "basic":false, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":true, day:"" ]
    ],
    "USECASE":[
            "FUNCvirNetNB" :                [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "FUNCbgpls" :                   [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "VPLSBasic" :                   [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "USECASE_SdnipFunction":        [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "USECASE_SdnipFunctionCluster": [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "PLATdockertest":               [ "basic":true, "extra_A":true, "extra_B":false, "new_Test":false, "day":"" ],
            "SRSanity":                     [ "basic":false, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "SRSwitchFailure":              [ "basic":false, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "SRLinkFailure":                [ "basic":false, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "SROnosFailure":                [ "basic":false, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "SRClusterRestart":             [ "basic":false, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "SRDynamic":                    [ "basic":false, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "SRHighAvailability":           [ "basic":false, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "VPLSfailsafe" :                [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ]
    ],
    "SR":[
            "SRBridging":                   [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ],
            "SRRouting":                   [ "basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":"" ]
    ]
]
testcases = [
    "FUNC" : [ tests : "" , nodeName : "VM" ],
    "HA" : [ tests : "" , nodeName : "VM" ],
    "SR" : [ tests : "", nodeName : "VM" ],
    "SCPF" : [ tests : "" , nodeName : "BM" ],
    "USECASE" : [ tests : "" , nodeName : "BM" ]
]
Prefix_organizer = [
    "FU" : "FUNC",
    "HA" : "HA",
    "PL" : "USECASE",
    "SA" : "USECASE",
    "SC" : "SCPF",
    "SR" : "SR",
    "US" : "USECASE",
    "VP" : "USECASE"
]

onos_b = "master"
test_branch = ""
onos_tag = params.ONOSTag
isOldFlow = true
// Set tests based on day of week
def now = funcs.getCurrentTime()
print now.toString()
today = now[ Calendar.DAY_OF_WEEK ]
day = ""
SCPF_choices = ""
USECASE_choices = ""
FUNC_choices = ""
HA_choices = ""
SR_choices = ""
stat_graph_generator_file = "testCategoryBuildStats.R"
pie_graph_generator_file = "testCategoryPiePassFail.R"
graph_saved_directory = "/var/jenkins/workspace/postjob-VM/"

manually_run = params.manual_run
post_result = params.PostResult
if( !manually_run ){
    slackSend( color:'#03CD9F',
               message:":sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles:\n"
                        + "Starting tests on : " + now.toString()
                        + "\n:sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles:" )
    testDivider( today )
    FUNC_choices =  lastCommaRemover( FUNC_choices )
    HA_choices =  lastCommaRemover( HA_choices )
    SCPF_choices =  lastCommaRemover( SCPF_choices )
    USECASE_choices =  lastCommaRemover( USECASE_choices )
    SR_choices =  lastCommaRemover( SR_choices )
}

if ( manually_run ){
    organize_tests( params.Tests )
    onos_b = params.ONOSVersion
    isOldFlow = params.isOldFlow
    println "Tests to be run manually : "
}else{
    testcases[ "SCPF" ][ "tests" ] = SCPF_choices
    testcases[ "USECASE" ][ "tests" ] = USECASE_choices
    testcases[ "FUNC" ][ "tests" ] = FUNC_choices
    testcases[ "HA" ][ "tests" ] = HA_choices
    testcases[ "SR" ][ "tests" ] = SR_choices
    println "Defaulting to " + day + " tests:"
}
print_tests( testcases )
def runTest = [
    "VM" : [:],
    "BM" : [:]
]
for( String test in testcases.keySet() ){
    println test
    if ( testcases[ test ][ "tests" ] != "" ){
        runTest[ testcases[ test ][ "nodeName" ] ][ test ] = trigger_pipeline( onos_b, testcases[ test ][ "tests" ], testcases[ test ][ "nodeName" ], test, manually_run, onos_tag )
    }
}
def finalList = [:]
finalList[ "VM" ] = runTestSeq( runTest[ "VM" ] )
finalList[ "BM" ] = runTestSeq( runTest[ "BM" ] )
parallel finalList
//finalList[ "BM" ].call()
generateStatGraph()

def testDivider( today ){
    switch ( today ) {
        case Calendar.MONDAY:
            monday( true )
            tuesday( true, false )
            wednesday( true, false )
            thursday( true, false )
            friday( true, false )
            day = "Monday"
            slackSend( color:'#FFD988', message:"Tests to be run this weekdays : \n" + printDaysForTest() )
            break
        case Calendar.TUESDAY:
            tuesday( true, true )
            day = "Tuesday"
            break
        case Calendar.WEDNESDAY:
            wednesday( true, true )
            day = "Wednesday"
            break
        case Calendar.THURSDAY:
            thursday( true, true )
            day = "Thursday"
            isOldFlow = false
            break
        case Calendar.FRIDAY:
            friday( true, true )
            day = "Friday"
            isOldFlow = false
            break
        case Calendar.SATURDAY:
            saturday()
            onos_b= previous_version
            day = "Saturday"
            break
        case Calendar.SUNDAY:
            sunday()
            onos_b= before_previous_version
            day = "Sunday"
            isOldFlow = false
            break
    }
}
def printDaysForTest(){
    result = ""
    for ( String test in AllTheTests.keySet() ){
        result += test + " : \n"
        for( String each in AllTheTests[ test ].keySet() ){
            AllTheTests[ test ][ each ][ "day" ] = lastCommaRemover( AllTheTests[ test ][ each ][ "day" ] )
            result += "    " + each + ":[" + AllTheTests[ test ][ each ][ "day" ] + "]\n"
        }
        result += "\n"
    }
    return result
}
def lastCommaRemover( str ){
    if ( str.size() > 0 && str[ str.size() - 1 ] == ',' ){
        str = str.substring( 0,str.size() - 1 )
    }
    return str
}
def monday( getResult ){
    FUNC_choices += adder( "FUNC", "basic", true, "M", getResult )
    FUNC_choices += adder( "FUNC", "new_Test", true, "M", getResult )
    FUNC_choices += adder( "FUNC", "extra_A", true, "M", getResult )
    HA_choices += adder( "HA", "basic", true, "M", getResult )
    HA_choices += adder( "HA", "extra_A", true, "M", getResult )
    //HA_choices += adder( "HA", "new_Test", true, "M", getResult )
    SR_choices += adder( "SR", "basic", true, "M", getResult )
    SCPF_choices += adder( "SCPF", "basic", true, "M", getResult )
    SCPF_choices += adder( "SCPF", "extra_B", true, "M", getResult )
}
def tuesday( getDay, getResult ){
    FUNC_choices += adder( "FUNC", "basic", getDay, "T", getResult )
    FUNC_choices += adder( "FUNC", "new_Test", getDay, "T", getResult )
    FUNC_choices += adder( "FUNC", "extra_B", getDay, "T", getResult )
    HA_choices += adder( "HA", "basic", getDay, "T", getResult )
    HA_choices += adder( "HA", "extra_B", getDay, "T", getResult )
    HA_choices += adder( "HA", "new_Test", getDay, "T", getResult )
    SR_choices += adder( "SR", "basic", getDay, "T", getResult )
    SCPF_choices += adder( "SCPF", "basic", getDay, "T", getResult )
    SCPF_choices += adder( "SCPF", "extra_C", getDay, "T", getResult )
    USECASE_choices += adder( "USECASE", "basic", getDay, "T", getResult )
    USECASE_choices += adder( "USECASE", "extra_A", getDay, "T", getResult )
    USECASE_choices += adder( "USECASE", "new_Test", getDay, "T", getResult )
}
def wednesday( getDay, getResult ){
    FUNC_choices += adder( "FUNC", "basic", getDay, "W", getResult )
    FUNC_choices += adder( "FUNC", "new_Test", getDay, "W", getResult )
    FUNC_choices += adder( "FUNC", "extra_A", getDay, "W", getResult )
    HA_choices += adder( "HA", "basic", getDay, "W", getResult )
    HA_choices += adder( "HA", "extra_A", getDay, "W", getResult )
    //HA_choices += adder( "HA", "new_Test", getDay, "W", getResult )
    SR_choices += adder( "SR", "basic", getDay, "W", getResult )
    SCPF_choices += adder( "SCPF", "basic", getDay, "W", getResult )
    SCPF_choices += adder( "SCPF", "extra_A", getDay, "W", getResult )
    SCPF_choices += adder( "SCPF", "new_Test", getDay, "W", getResult )
}
def thursday( getDay, getResult ){
    FUNC_choices += adder( "FUNC", "basic", getDay, "Th", getResult )
    FUNC_choices += adder( "FUNC", "new_Test", getDay, "Th", getResult )
    FUNC_choices += adder( "FUNC", "extra_B", getDay, "Th", getResult )
    HA_choices += adder( "HA", "basic", getDay, "Th", getResult )
    HA_choices += adder( "HA", "extra_B", getDay, "Th", getResult )
    HA_choices += adder( "HA", "new_Test", getDay, "Th", getResult )
    SR_choices += adder( "SR", "basic", getDay, "Th", getResult )
    SCPF_choices += adder( "SCPF", "basic", getDay, "Th", getResult )
    SCPF_choices += adder( "SCPF", "extra_B", getDay, "Th", getResult )
}
def friday( getDay, getResult ){
    FUNC_choices += adder( "FUNC", "basic", getDay, "F", getResult )
    FUNC_choices += adder( "FUNC", "new_Test", getDay, "F", getResult )
    FUNC_choices += adder( "FUNC", "extra_A", getDay, "F", getResult )
    HA_choices += adder( "HA", "basic", getDay, "F", getResult )
    HA_choices += adder( "HA", "extra_A", getDay, "F", getResult )
    //HA_choices += adder( "HA", "new_Test", getDay, "F", getResult )
    SR_choices += adder( "SR", "basic", getDay, "F", getResult )
    SCPF_choices += adder( "SCPF", "basic", getDay, "F", getResult )
    SCPF_choices += adder( "SCPF", "extra_A", getDay, "F", getResult )
    SCPF_choices += adder( "SCPF", "extra_D", getDay, "F", getResult )
}
def saturday(){
    FUNC_choices += adder( "FUNC", "basic", false, "Sa", true )
    FUNC_choices += adder( "FUNC", "extra_A", false, "Sa", true )
    FUNC_choices += adder( "FUNC", "extra_B", false, "Sa", true )
    FUNC_choices += adder( "FUNC", "new_Test", true, "Sa", true )
    HA_choices += adder( "HA", "basic", false, "Sa", true )
    HA_choices += adder( "HA", "extra_A", false, "Sa", true )
    HA_choices += adder( "HA", "extra_B", false, "Sa", true )
    HA_choices += adder( "HA", "new_Test", false, "Sa", true )
    SR_choices += adder( "SR", "basic", false, "Sa", true )
    SCPF_choices += adder( "SCPF", "basic", false, "Sa", true )
    SCPF_choices += adder( "SCPF", "extra_A", false, "Sa", true )
    SCPF_choices += adder( "SCPF", "extra_B", false, "Sa", true )
    SCPF_choices += adder( "SCPF", "extra_C", false, "Sa", true )
    SCPF_choices += adder( "SCPF", "extra_D", false, "Sa", true )
    SCPF_choices += adder( "SCPF", "new_Test", false, "Sa", true )
    USECASE_choices += adder( "USECASE", "basic", false, "Sa", true )
}
def sunday(){
    FUNC_choices += adder( "FUNC", "basic", false, "S", true )
    FUNC_choices += adder( "FUNC", "extra_A", false, "S", true )
    FUNC_choices += adder( "FUNC", "extra_B", false, "S", true )
    HA_choices += adder( "HA", "basic", false, "S", true )
    HA_choices += adder( "HA", "extra_A", false, "S", true )
    HA_choices += adder( "HA", "extra_B", false, "S", true )
    SR_choices += adder( "SR", "basic", false, "S", true )
    SCPF_choices += adder( "SCPF", "basic", false, "S", true )
    USECASE_choices += adder( "USECASE", "basic", false, "S", true )
}
def adder( testCat, set, dayAdding, day, getResult ){
    result = ""
    for( String test in AllTheTests[ testCat ].keySet()  ){
        if( AllTheTests[ testCat ][ test ][ set ] ){
            if( getResult )
                result += test + ","
            if( dayAdding )
                dayAdder( testCat, test, day )
        }
    }
    return result
}
def dayAdder( testCat, testName, dayOfWeek ){
    AllTheTests[ testCat ][ testName ][ "day" ] += dayOfWeek + ","
}
def runTestSeq( testList ){
    return{
        for ( test in testList.keySet() ){
            testList[ test ].call()
        }
    }
}

def print_tests( tests ){
    for( String test in tests.keySet() ){
        if( tests[ test ][ "tests" ] != "" ){
            println test + ":"
            println tests[ test ][ "tests" ]
        }
    }
}
def organize_tests( tests ){
    testList = tests.tokenize( "\n;, " )
    for( String test in testList )
        testcases [ Prefix_organizer[ ( test == "FUNCbgpls" || test == "FUNCvirNetNB" ? "US" : ( test[ 0 ] + test[ 1 ] ) ) ] ][ "tests" ] += test + ","
}
def borrow_mn( jobOn ){
    result = ""
    if( jobOn == "SR" ){
        result = "~/cell_borrow.sh"
    }
    return result
}
def trigger( branch, tests, nodeName, jobOn, manuallyRun, onosTag ){
    println jobOn + "-pipeline-" + manuallyRun ? "manually" : branch
    wiki = branch
    if ( branch != "master" ){
        branch = "onos-" + branch
    }
    test_branch = "master"
    node( "TestStation-" + nodeName + "s" ){
        envSetup( branch, test_branch, onosTag, jobOn, manuallyRun )

        exportEnvProperty( branch, test_branch, wiki, tests, post_result, manuallyRun, onosTag, isOldFlow )
    }

    jobToRun = jobOn + "-pipeline-" + ( manuallyRun ? "manually" : wiki )
    build job: jobToRun, propagate: false
}
def trigger_pipeline( branch, tests, nodeName, jobOn, manuallyRun, onosTag ){
// nodeName : "BM" or "VM"
// jobOn : "SCPF" or "USECASE" or "FUNC" or "HA"
    return{
        if( jobOn == "SR" ){
            trigger( "1.11", "SRBridging", nodeName, jobOn, manuallyRun, onosTag )
            trigger( "1.12", "SRBridging", nodeName, jobOn, manuallyRun, onosTag )
            trigger( "master", "SRBridging", nodeName, jobOn, manuallyRun, onosTag )
            // returnCell( nodeName )
            }else{
            trigger( branch, tests, nodeName, jobOn, manuallyRun, onosTag )
        }
    }
}

// export Environment properties.
def exportEnvProperty( onos_branch, test_branch, wiki, tests, postResult, manually_run, onosTag, isOldFlow ){
    stage( "export Property" ){
        sh '''
            echo "ONOSBranch=''' + onos_branch +'''" > /var/jenkins/TestONOS.property
            echo "TestONBranch=''' + test_branch +'''" >> /var/jenkins/TestONOS.property
            echo "ONOSTag='''+ onosTag +'''" >> /var/jenkins/TestONOS.property
            echo "WikiPrefix=''' + wiki +'''" >> /var/jenkins/TestONOS.property
            echo "ONOSJVMHeap='''+ env.ONOSJVMHeap +'''" >> /var/jenkins/TestONOS.property
            echo "Tests=''' + tests +'''" >> /var/jenkins/TestONOS.property
            echo "postResult=''' + postResult +'''" >> /var/jenkins/TestONOS.property
            echo "manualRun=''' + manually_run +'''" >> /var/jenkins/TestONOS.property
            echo "isOldFlow=''' + isOldFlow +'''" >> /var/jenkins/TestONOS.property

        '''
    }
}
// Initialize the environment Setup for the onos and OnosSystemTest
def envSetup( onos_branch, test_branch, onos_tag, jobOn, manuallyRun ){
    stage( "envSetup" ) {
        sh '''#!/bin/bash -l
        set +e
        . ~/.bashrc
        env
        ''' //+ borrow_mn( jobOn )
        + '''
        ''' + preSetup( onos_branch, test_branch, onos_tag, manuallyRun ) + '''
        ''' + oldFlowCheck( jobOn, onos_branch ) + '''
        ''' + postSetup( onos_branch, test_branch, onos_tag, manuallyRun )
    }
}
def tagCheck( onos_tag, onos_branch ){
    result = "git checkout "
    if ( onos_tag == "" )
        result += onos_branch //create new local branch
    else
        result += onos_tag //checkout the tag
    return result
}
def preSetup( onos_branch, test_branch, onos_tag, isManual ){
    result = ""
    if( !isManual ){
        result = '''echo -e "\n#####  Set TestON Branch #####"
        echo "TestON Branch is set on: ''' + test_branch + '''"

        cd ~/OnosSystemTest/
        git checkout HEAD~1      # Make sure you aren't pn a branch
        git branch | grep -v "detached from" | xargs git branch -d # delete all local branches merged with remote
        git branch -D ''' + test_branch + ''' # just incase there are local changes. This will normally result in a branch not found error
        git clean -df # clean any local files
        git fetch --all # update all caches from remotes
        git reset --hard origin/''' + test_branch +'''  # force local index to match remote branch
        git clean -df # clean any local files
        git checkout ''' + test_branch + ''' #create new local branch
        git branch
        git log -1 --decorate


        echo -e "\n#####  Set ONOS Branch #####"
        echo "ONOS Branch is set on: ''' + onos_branch + '''"

        echo -e "\n #### check karaf version ######"
        env |grep karaf

        cd ~/onos
        rm -rf buck-out/*
        ~/onos/tools/build/onos-buck clean
        git checkout HEAD~1      # Make sure you aren't pn a branch
        git branch | grep -v "detached from" | xargs git branch -d # delete all local branches merged with remote
        git branch -D ''' + onos_branch + ''' # just incase there are local changes. This will normally result in a branch not found error
        git clean -df # clean any local files
        git fetch --all # update all caches from remotes
        git reset --hard origin/''' + onos_branch + '''  # force local index to match remote branch
        git clean -df # clean any local files
        ''' + tagCheck( onos_tag, onos_branch ) + '''
        git branch
        git log -1 --decorate


        echo -e "\n##### set jvm heap size to 8G #####"
        echo ${ONOSJVMHeap}

        inserted_line="export JAVA_OPTS=\"\${ONOSJVMHeap}\""
        sed -i "s/bash/bash\\n$inserted_line/" ~/onos/tools/package/bin/onos-service

        echo "##### Check onos-service setting..... #####"
        cat ~/onos/tools/package/bin/onos-service

        export JAVA_HOME=/usr/lib/jvm/java-8-oracle'''
    }
    return result
}
def oldFlowCheck( jobOn, onos_branch ){
    result = ""
    if( jobOn == "SCPF" && ( onos_branch== "master" || onos_branch=="onos-1.12" ) )
        result = '''sed -i -e 's/@Component(immediate = true)/@Component(enabled = false)/g' ~/onos/core/store/dist/src/main/java/org/onosproject/store/flow/impl/''' + ( isOldFlow ? "DistributedFlowRuleStore" : "ECFlowRuleStore" ) + '''.java
        sed -i -e 's/@Component(enabled = false)/@Component(immediate = true)/g' ~/onos/core/store/dist/src/main/java/org/onosproject/store/flow/impl/''' + ( isOldFlow ? "ECFlowRuleStore" : "DistributedFlowRuleStore" ) + ".java"
    return result
}
def postSetup( onos_branch, test_branch, onos_tag, isManual ){
    result = ""
    if( !isManual ){
        result = '''echo -e "\n##### build ONOS skip unit tests ######"
        #mvn clean install -DskipTests
        # Force buck update
        rm -f ~/onos/bin/buck
        ~/onos/tools/build/onos-buck build onos

        sleep 30
        echo -e "\n##### Stop all running instances of Karaf #####"
        kill $(ps -efw | grep karaf | grep -v grep | awk '{print $2}')
        sleep 30

        git branch'''
    }
    return result
}
def returnCell( nodeName ){
    node( "TestStation-" + nodeName + "s" ){
        sh '''#!/bin/bash -l
            set +e
            . ~/.bashrc
            env
            ~/./return_cell.sh
            '''
    }
}

def generateStatGraph(){
    if( !manually_run ){
        testListPart = funcs.createStatsList( "FUNC", AllTheTests[ "FUNC" ], true ) +
                        funcs.createStatsList( "HA", AllTheTests[ "HA" ], true ) +
                        funcs.createStatsList( "USECASE", AllTheTests[ "USECASE" ], false )
        pieTestList = funcs.makeTestList( AllTheTests[ "FUNC" ], true ) +
                      funcs.makeTestList( AllTheTests[ "HA" ], true ) +
                      funcs.makeTestList( AllTheTests[ "USECASE" ], false )
        funcs.generateCategoryStatsGraph( "false", "true", stat_graph_generator_file, pie_graph_generator_file, "ALL", onos_b, testListPart, graph_saved_directory, pieTestList )
    }
}