Merge "[ONOS-7101] Integrate Jenkins to Slack to notify information"
diff --git a/TestON/JenkinsFile/FUNCJenkinsFile b/TestON/JenkinsFile/FUNCJenkinsFile
index e7b1d3b..6b6112d 100644
--- a/TestON/JenkinsFile/FUNCJenkinsFile
+++ b/TestON/JenkinsFile/FUNCJenkinsFile
@@ -1,4 +1,5 @@
 #!groovy
+import groovy.time.*
 // This is a Jenkinsfile for a scripted pipeline for the FUNC tests
 def prop = null
 node("TestStation-VMs"){
@@ -34,15 +35,20 @@
     tests[stepName] = FUNCTest(test, toBeRun, prop)
 }
 
+def now = new Date()
 // run the tests
 for ( test in tests.keySet() ){
     tests[test].call()
 }
-
+if( prop["manualRun"] == "false" ){
+    def end = new Date()
+    TimeDuration duration = TimeCategory.minus( end, now )
+    slackSend( color:"#5816EE", message: "FUNC tests ended at: " + end.toString() + "\nTime took : " + duration )
+}
 // The testName should be the key from the FUNC
 def FUNCTest( testName, toBeRun, prop ) {
     return {
-        catchError{
+        try{
             stage(testName) {
                 if ( toBeRun ){
                     workSpace = "/var/jenkins/workspace/"+testName
@@ -174,6 +180,12 @@
                     }
                 }
             }
+        }catch (all) {
+            catchError{
+                if( prop["manualRun"] == "false" )
+                    slackSend(color:"FF0000", message: "[" + prop["TestONBranch"] + "]" + testName + " : Failed!\nURL:${BUILD_URL}")
+                Failed
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/TestON/JenkinsFile/HAJenkinsFile b/TestON/JenkinsFile/HAJenkinsFile
index d123075..2f63e62 100644
--- a/TestON/JenkinsFile/HAJenkinsFile
+++ b/TestON/JenkinsFile/HAJenkinsFile
@@ -1,4 +1,5 @@
 #!groovy
+import groovy.time.*
 // This is a Jenkinsfile for a scripted pipeline for the HA tests
 
 def prop = null
@@ -35,16 +36,20 @@
     tests[stepName] = HATest(test, toBeRun, prop)
 }
 
+def now = new Date()
 // run the tests
 for ( test in tests.keySet() ){
     tests[test].call()
 }
-
-
+if( prop["manualRun"] == "false" ){
+    def end = new Date()
+    TimeDuration duration = TimeCategory.minus( end, now )
+    slackSend( color:"#5816EE", message: "HA tests ended at: " + end.toString() + "\nTime took : " + duration )
+}
 // The testName should be the key from the FUNC
 def HATest( testName, toBeRun, prop ) {
     return {
-        catchError{
+        try{
             stage(testName) {
                 if ( toBeRun ){
                     workSpace = "/var/jenkins/workspace/"+testName
@@ -111,7 +116,7 @@
                             ls -al
                             cd '''
 
-                            if( prop["manualRun"] == "false" ){
+                            if( prop["manualRun"] == "false" || prop["postResult"] == "true" ){
                                 // Post Results
                                 withCredentials([
                                     string(credentialsId: 'db_pass', variable: 'pass'),
@@ -156,7 +161,7 @@
                         }
                     }
 
-                    if( prop["manualRun"] == "false" ){
+                    if( prop["manualRun"] == "false" || prop["postResult"] == "true" ){
                         def post = build job: "Pipeline_postjob_VM", propagate: false,
                             parameters: [
                                 string(name: 'Wiki_Contents', value: fileContents),
@@ -176,6 +181,12 @@
                     }
                 }
             }
+        }catch (all) {
+            catchError{
+                if( prop["manualRun"] == "false" )
+                    slackSend(color:"FF0000", message: "[" + prop["TestONBranch"] + "]" + testName + " : Failed!\nURL:${BUILD_URL}")
+                Failed
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/TestON/JenkinsFile/JenkinsfileTrigger b/TestON/JenkinsFile/JenkinsfileTrigger
index c20db0c..fd856eb 100644
--- a/TestON/JenkinsFile/JenkinsfileTrigger
+++ b/TestON/JenkinsFile/JenkinsfileTrigger
@@ -1,18 +1,66 @@
 #!groovy
 // This is a Jenkinsfile for a scripted pipeline for the SCPF tests
 // Define sets of tests
-SCPF_Basic = "SCPFswitchLat,SCPFportLat,SCPFintentInstallWithdrawLat,SCPFintentEventTp,SCPFflowTp1g,SCPFcbench,SCPFbatchFlowResp"
-SCPF_ExtraSetA = "SCPFintentRerouteLat,SCPFscalingMaxIntents,SCPFhostLat,SCPFscaleTopo,SCPFmastershipFailoverLat"
-SCPF_NEW_TEST = "SCPFmastershipFailoverLat"
-SCPF_ExtraSetB = "SCPFintentInstallWithdrawLatWithFlowObj,SCPFintentEventTpWithFlowObj,SCPFintentRerouteLatWithFlowObj,SCPFscalingMaxIntentsWithFlowObj,SCPFflowTp1gWithFlowObj"
-FUNC_Basic = "FUNCipv6Intent,FUNCoptical,FUNCflow,FUNCnetCfg,FUNCovsdbtest,FUNCnetconf"
-FUNC_ExtraSetA = "FUNCgroup,FUNCintent"
-FUNC_ExtraSetB = "FUNCintentRest"
-HA_Basic = "HAsanity,HAsingleInstanceRestart,HAclusterRestart"
-HA_ExtraSetA = "HAstopNodes,HAfullNetPartition"
-HA_ExtraSetB = "HAkillNodes,HAswapNodes,HAscaling"
-USECASE_Basic = "FUNCvirNetNB,FUNCbgpls,VPLSBasic,PLATdockertest,SRSanity,SRSwitchFailure,SRLinkFailure,SROnosFailure,SRClusterRestart,SRDynamic,SRHA,USECASE_SdnipFunction,USECASE_SdnipFunctionCluster"
-USECASE_NEW_TEST = "VPLSfailsafe"
+AllTheTests=
+[
+    "FUNC":[
+            "FUNCipv6Intent" : ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "FUNCoptical" :    ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "FUNCflow" :       ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "FUNCnetCfg":      ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "FUNCovsdbtest" :  ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "FUNCnetconf" :    ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "FUNCgroup" :      ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "FUNCintent" :     ["basic":false, "extra_A":true, "extra_B":false, "day":""],
+            "FUNCintentRest" : ["basic":false, "extra_A":false, "extra_B":true, "day":""]
+    ],
+    "HA":[
+            "HAsanity" :                ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "HAclusterRestart" :        ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "HAsingleInstanceRestart" : ["basic":true, "extra_A":false, "extra_B":false, "day":""],
+            "HAstopNodes" :             ["basic":false, "extra_A":true, "extra_B":false, "day":""],
+            "HAfullNetPartition" :      ["basic":false, "extra_A":true, "extra_B":false, "day":""],
+            "HAswapNodes" :             ["basic":false, "extra_A":false, "extra_B":true, "day":""],
+            "HAscaling" :               ["basic":false, "extra_A":false, "extra_B":true, "day":""],
+            "HAkillNodes" :             ["basic":false, "extra_A":false, "extra_B":true, "day":""]
+    ],
+    "SCPF":[
+            "SCPFswitchLat":                           ["basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFcbench":                              ["basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFportLat":                             ["basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFflowTp1g":                            ["basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFintentEventTp":                       ["basic":true, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFhostLat":                             ["basic":false, "extra_A":true, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            // batch will be on extra_A after fixing from the ONOS side.
+            "SCPFbatchFlowResp":                       ["basic":false, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFintentRerouteLat":                    ["basic":false, "extra_A":true, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFintentInstallWithdrawLat":            ["basic":false, "extra_A":true, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFflowTp1gWithFlowObj":                 ["basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFintentEventTpWithFlowObj":            ["basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFintentRerouteLatWithFlowObj":         ["basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFscalingMaxIntentsWithFlowObj":        ["basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFintentInstallWithdrawLatWithFlowObj": ["basic":false, "extra_A":false, "extra_B":true, "extra_C":false, "extra_D":false, "new_Test":false, day:""],
+            "SCPFscaleTopo":                           ["basic":false, "extra_A":false, "extra_B":false, "extra_C":true, "extra_D":false, "new_Test":false, day:""],
+            "SCPFscalingMaxIntents":                   ["basic":false, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":true, "new_Test":false, day:""],
+            "SCPFmastershipFailoverLat":               ["basic":false, "extra_A":false, "extra_B":false, "extra_C":false, "extra_D":false, "new_Test":true, day:""]
+    ],
+    "USECASE":[
+            "FUNCvirNetNB" :                ["basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":""],
+            "FUNCbgpls" :                   ["basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":""],
+            "VPLSBasic" :                   ["basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":""],
+            "USECASE_SdnipFunction":        ["basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":""],
+            "USECASE_SdnipFunctionCluster": ["basic":true, "extra_A":false, "extra_B":false, "new_Test":false, "day":""],
+            "PLATdockertest":               ["basic":false, "extra_A":true, "extra_B":false, "new_Test":false, "day":""],
+            "SRSanity":                     ["basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":""],
+            "SRSwitchFailure":              ["basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":""],
+            "SRLinkFailure":                ["basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":""],
+            "SROnosFailure":                ["basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":""],
+            "SRClusterRestart":             ["basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":""],
+            "SRDynamic":                    ["basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":""],
+            "SRHighAvailability":           ["basic":false, "extra_A":false, "extra_B":true, "new_Test":false, "day":""],
+            "VPLSfailsafe" :                ["basic":false, "extra_A":false, "extra_B":false, "new_Test":true, "day":""]
+    ]
+]
 testcases = [
     "FUNC" : [ tests : "" , nodeName : "VM"],
     "HA" : [ tests : "" , nodeName : "VM"],
@@ -37,62 +85,24 @@
 echo(now.toString())
 today = now[Calendar.DAY_OF_WEEK]
 day = ""
-SCPF_choices = SCPF_Basic
+SCPF_choices = ""
 USECASE_choices = ""
-FUNC_choices = FUNC_Basic
-HA_choices = HA_Basic
-
-switch (today) {
-    case Calendar.MONDAY:
-        SCPF_choices += "," + SCPF_ExtraSetB
-        FUNC_choices += "," + FUNC_ExtraSetA
-        HA_choices += "," + HA_ExtraSetA
-        day = "Monday"
-        break
-    case Calendar.TUESDAY:
-        day = "Tuesday"
-        USECASE_choices = USECASE_Basic + "," + USECASE_NEW_TEST
-        FUNC_choices += "," + FUNC_ExtraSetB
-        HA_choices += "," + HA_ExtraSetB
-        break
-    case Calendar.WEDNESDAY:
-        SCPF_choices += "," + SCPF_ExtraSetA + "," + SCPF_NEW_TEST
-        FUNC_choices += "," + FUNC_ExtraSetA
-        HA_choices += "," + HA_ExtraSetA
-        day = "Wednesday"
-        break
-    case Calendar.THURSDAY:
-        SCPF_choices += "," + SCPF_ExtraSetB
-        FUNC_choices += "," + FUNC_ExtraSetB
-        HA_choices += "," + HA_ExtraSetB
-        day = "Thursday"
-        break
-    case Calendar.FRIDAY:
-        SCPF_choices += "," + SCPF_ExtraSetA + "," + SCPF_NEW_TEST + "," + SCPF_ExtraSetB
-        FUNC_choices += "," + FUNC_ExtraSetA
-        HA_choices += "," + HA_ExtraSetA
-        day = "Friday"
-        break
-    case Calendar.SATURDAY:
-        SCPF_choices += "," + SCPF_ExtraSetA  + "," + SCPF_ExtraSetB
-        USECASE_choices = USECASE_Basic
-        FUNC_choices += "," + FUNC_ExtraSetA + "," + FUNC_ExtraSetB
-        HA_choices += "," + HA_ExtraSetA + "," + HA_ExtraSetB
-        onos_branch= "1.10"
-        day = "Saturday"
-        break
-    case Calendar.SUNDAY:
-        SCPF_choices += "," + SCPF_ExtraSetA + "," + SCPF_NEW_TEST + "," + SCPF_ExtraSetB
-        USECASE_choices = USECASE_Basic + "," + USECASE_NEW_TEST
-        FUNC_choices += "," + FUNC_ExtraSetA + "," + FUNC_ExtraSetB
-        HA_choices += "," + HA_ExtraSetA + "," + HA_ExtraSetB
-        onos_branch= "1.11"
-        day = "Sunday"
-        break
-}
+FUNC_choices = ""
+HA_choices = ""
 
 manually_run = params.manual_run
 post_result = params.PostResult
+if( !manually_run ){
+    sendToSlack( '#03CD9F', ":sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles:\n"
+                            + "Starting tests on : " + now.toString()
+                            + "\n:sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles::sparkles:" )
+    testDivider( today )
+    FUNC_choices =  lastCommaRemover( FUNC_choices )
+    HA_choices =  lastCommaRemover( HA_choices )
+    SCPF_choices =  lastCommaRemover( SCPF_choices )
+    USECASE_choices =  lastCommaRemover( USECASE_choices )
+}
+
 if ( manually_run ){
     organize_tests( params.Tests )
     onos_branch = params.ONOSVersion
@@ -107,6 +117,149 @@
 }
 print_tests( testcases )
 
+def testDivider( today ){
+    switch (today) {
+        case Calendar.MONDAY:
+            monday( true )
+            tuesday( true, false )
+            wednesday( true, false )
+            thursday( true, false )
+            friday( true, false )
+            day = "Monday"
+            sendToSlack( '#FFD988', "Tests to be run this weekdays : \n" + printDaysForTest() )
+            break
+        case Calendar.TUESDAY:
+            tuesday( true, true )
+            day = "Tuesday"
+            break
+        case Calendar.WEDNESDAY:
+            wednesday( true, true )
+            day = "Wednesday"
+            break
+        case Calendar.THURSDAY:
+            thursday( true, true)
+            day = "Thursday"
+            break
+        case Calendar.FRIDAY:
+            friday( true, true)
+            day = "Friday"
+            break
+        case Calendar.SATURDAY:
+            saturday()
+            onos_branch= "1.11"
+            day = "Saturday"
+            break
+        case Calendar.SUNDAY:
+            sunday()
+            onos_branch= "1.10"
+            day = "Sunday"
+            break
+    }
+}
+def printDaysForTest(){
+    result = ""
+    for ( String test in AllTheTests.keySet() ){
+        result += test + " : \n"
+        for( String each in AllTheTests[ test ].keySet() ){
+            AllTheTests[ test ][ each ][ "day" ] = lastCommaRemover( AllTheTests[ test ][ each ][ "day" ] )
+            result += "    " + each + ":[" + AllTheTests[ test ][ each ][ "day" ] + "]\n"
+        }
+        result += "\n"
+    }
+    return result
+}
+def lastCommaRemover( str ){
+    if ( str.size() > 0 && str[ str.size() - 1 ] == ',' ){
+        str = str.substring( 0,str.size() - 1 )
+    }
+    return str
+}
+def monday( getResult ){
+    FUNC_choices += adder( "FUNC", "basic", true, "M", getResult )
+    FUNC_choices += adder( "FUNC", "extra_A", true, "M", getResult )
+    HA_choices += adder( "HA", "basic", true, "M", getResult )
+    HA_choices += adder( "HA", "extra_A", true, "M", getResult )
+    SCPF_choices += adder( "SCPF", "basic", true, "M", getResult )
+    SCPF_choices += adder( "SCPF", "extra_B", true, "M", getResult )
+}
+def tuesday( getDay, getResult ){
+    FUNC_choices += adder( "FUNC", "basic", getDay, "T", getResult )
+    FUNC_choices += adder( "FUNC", "extra_B", getDay, "T", getResult )
+    HA_choices += adder( "HA", "basic", getDay, "T", getResult )
+    HA_choices += adder( "HA", "extra_B", getDay, "T", getResult )
+    SCPF_choices += adder( "SCPF", "basic", getDay, "T", getResult )
+    SCPF_choices += adder( "SCPF", "extra_C", getDay, "T", getResult )
+    USECASE_choices += adder( "USECASE", "basic", getDay, "T", getResult )
+    USECASE_choices += adder( "USECASE", "extra_A", getDay, "T", getResult )
+    USECASE_choices += adder( "USECASE", "new_Test", getDay, "T", getResult )
+}
+def wednesday( getDay, getResult ){
+    FUNC_choices += adder( "FUNC", "basic", getDay, "W", getResult )
+    FUNC_choices += adder( "FUNC", "extra_A", getDay, "W", getResult )
+    HA_choices += adder( "HA", "basic", getDay, "W", getResult )
+    HA_choices += adder( "HA", "extra_A", getDay, "W", getResult )
+    SCPF_choices += adder( "SCPF", "basic", getDay, "W", getResult )
+    SCPF_choices += adder( "SCPF", "extra_A", getDay, "W", getResult )
+    SCPF_choices += adder( "SCPF", "new_Test", getDay, "W", getResult )
+}
+def thursday( getDay, getResult ){
+    FUNC_choices += adder( "FUNC", "basic", getDay, "Th", getResult )
+    FUNC_choices += adder( "FUNC", "extra_B", getDay, "Th", getResult )
+    HA_choices += adder( "HA", "basic", getDay, "Th", getResult )
+    HA_choices += adder( "HA", "extra_B", getDay, "Th", getResult )
+    SCPF_choices += adder( "SCPF", "basic", getDay, "Th", getResult )
+    SCPF_choices += adder( "SCPF", "extra_B", getDay, "Th", getResult )
+}
+def friday( getDay, getResult ){
+    FUNC_choices += adder( "FUNC", "basic", getDay, "F", getResult )
+    FUNC_choices += adder( "FUNC", "extra_A", getDay, "F", getResult )
+    HA_choices += adder( "HA", "basic", getDay, "F", getResult )
+    HA_choices += adder( "HA", "extra_A", getDay, "F", getResult )
+    SCPF_choices += adder( "SCPF", "basic", getDay, "F", getResult )
+    SCPF_choices += adder( "SCPF", "extra_A", getDay, "F", getResult )
+    SCPF_choices += adder( "SCPF", "extra_D", getDay, "F", getResult )
+}
+def saturday(){
+    FUNC_choices += adder( "FUNC", "basic", false, "Sa", true )
+    FUNC_choices += adder( "FUNC", "extra_A", false, "Sa", true )
+    FUNC_choices += adder( "FUNC", "extra_B", false, "Sa", true )
+    HA_choices += adder( "HA", "basic", false, "Sa", true )
+    HA_choices += adder( "HA", "extra_A", false, "Sa", true )
+    HA_choices += adder( "HA", "extra_B", false, "Sa", true )
+    SCPF_choices += adder( "SCPF", "basic", false, "Sa", true )
+    SCPF_choices += adder( "SCPF", "extra_A", false, "Sa", true )
+    SCPF_choices += adder( "SCPF", "extra_B", false, "Sa", true )
+    SCPF_choices += adder( "SCPF", "extra_C", false, "Sa", true )
+    SCPF_choices += adder( "SCPF", "extra_D", false, "Sa", true )
+    SCPF_choices += adder( "SCPF", "new_Test", false, "Sa", true )
+    USECASE_choices += adder( "USECASE", "basic", false, "Sa", true )
+    USECASE_choices += adder( "USECASE", "new_Test", false, "Sa", true )
+}
+def sunday(){
+    FUNC_choices += adder( "FUNC", "basic", false, "S", true )
+    FUNC_choices += adder( "FUNC", "extra_A", false, "S", true )
+    FUNC_choices += adder( "FUNC", "extra_B", false, "S", true )
+    HA_choices += adder( "HA", "basic", false, "S", true )
+    HA_choices += adder( "HA", "extra_A", false, "S", true )
+    HA_choices += adder( "HA", "extra_B", false, "S", true )
+    SCPF_choices += adder( "SCPF", "basic", false, "S", true )
+    USECASE_choices += adder( "USECASE", "basic", false, "S", true )
+}
+def adder( testCat, set, dayAdding, day, getResult ){
+    result = ""
+    for( String test in AllTheTests[ testCat ].keySet()  ){
+        if( AllTheTests[ testCat ][ test ][ set ] ){
+            if( getResult )
+                result += test + ","
+            if( dayAdding )
+                dayAdder( testCat, test, day )
+        }
+    }
+    return result
+}
+def dayAdder( testCat, testName, dayOfWeek ){
+    AllTheTests[ testCat ][ testName ][ "day" ] += dayOfWeek + ","
+}
 def runTest = [
     "VM" : [:],
     "BM" : [:]
@@ -185,6 +338,9 @@
         '''
     }
 }
+def sendToSlack( color, message ){
+    slackSend(color:color, message: message)
+}
 // Initialize the environment Setup for the onos and OnosSystemTest
 def envSetup( onos_branch, test_branch, onos_tag ){
     stage("envSetup") {
diff --git a/TestON/JenkinsFile/SCPFJenkinsFile b/TestON/JenkinsFile/SCPFJenkinsFile
index 2661cca..297c91b 100644
--- a/TestON/JenkinsFile/SCPFJenkinsFile
+++ b/TestON/JenkinsFile/SCPFJenkinsFile
@@ -1,4 +1,5 @@
 #!groovy
+import groovy.time.*
 // This is a Jenkinsfile for a scripted pipeline for the SCPF tests
 // properties([pipelineTriggers([cron('30 19 * * *')])])
 
@@ -7,6 +8,7 @@
 none = [ "" ]
 batches = [1,100,1000]
 neighbors = ['y', 'n']
+times = [ 'y', 'n' ]
 SCPF = [
     SCPFcbench: [ test:'SCPFcbench', table:'cbench_bm_tests', results:'cbench_bm_results', file:'CbenchDB', rFile:'SCPFcbench.R', extra:none, finalResult:1, graphTitle:['Cbench Test'], dbCols:'avg', dbWhere:'', y_axis:'Throughput (Responses/sec)'],
     SCPFhostLat: [ test:'SCPFhostLat', table:'host_latency_tests', results:'host_latency_results', file:'HostAddLatency', rFile:'SCPFhostLat.R', extra:none,finalResult:1, graphTitle:['Host Latency Test'], dbCols:'avg', dbWhere:'AND scale=5', y_axis:'Latency (ms)'],
@@ -18,11 +20,11 @@
     SCPFbatchFlowResp: [ test:'SCPFbatchFlowResp', table:'batch_flow_tests', results:'batch_flow_results', file:'SCPFbatchFlowRespData', rFile:'SCPFbatchFlowResp.R', extra:none, finalResult:1, graphTitle:['Batch Flow Test - Post', 'Batch Flow Test - Del'], dbCols:[ 'elapsepost, posttoconfrm', 'elapsedel, deltoconfrm' ], dbWhere:'', y_axis:'Latency (ms)'],
     SCPFintentEventTp: [ test:'SCPFintentEventTp', table:'intent_tp_tests', results:'intent_tp_results', file:'IntentEventTPDB', rFile:'SCPFintentEventTp.R n', extra:neighbors, finalResult:1, graphTitle:['Intent Throughput Test - neighbors=0','Intent Throughput Test - neighbors=4'], dbCols:'SUM( avg ) as avg', dbWhere:[ 'AND scale=5 AND neighbors=0 GROUP BY date','AND scale=5 AND NOT neighbors=0 GROUP BY date' ], y_axis:'Throughput (Ops/sec)'],
     SCPFintentRerouteLat: [ test:'SCPFintentRerouteLat', table:'intent_reroute_latency_tests', results:'intent_reroute_latency_results', file:'IntentRerouteLatDB', rFile:'SCPFIntentInstallWithdrawRerouteLat.R n', extra:batches, finalResult:1, graphTitle:['Intent Reroute Test'], dbCols:'avg', dbWhere:'AND scale=5 AND batch_size=100', y_axis:'Latency (ms)'],
-    SCPFscalingMaxIntents: [ test:'SCPFscalingMaxIntents', table:'max_intents_tests', results:'max_intents_results', file:'ScalingMaxIntentDB', rFile:'SCPFscalingMaxIntents.R n', extra:none, finalResult:0],
+    SCPFscalingMaxIntents: [ test:'SCPFscalingMaxIntents', table:'max_intents_tests', results:'max_intents_results', file:'ScalingMaxIntentDB', rFile:'SCPFscalingMaxIntents.R n', extra:times, finalResult:0],
     SCPFintentEventTpWithFlowObj: [ test:'SCPFintentEventTp --params TEST/flowObj=True', table:'intent_tp_fobj_tests', results:'intent_tp_fobj_results', file:'IntentEventTPflowObjDB', rFile:'SCPFintentEventTp.R y', extra:neighbors,finalResult:0],
     SCPFintentInstallWithdrawLat: [ test:'SCPFintentInstallWithdrawLat', table:'intent_latency_tests', results:'intent_latency_results', file:'IntentInstallWithdrawLatDB', rFile:'SCPFIntentInstallWithdrawRerouteLat.R n', extra:batches,finalResult:1, graphTitle:['Intent Installation Test','Intent Withdrawal Test'], dbCols:[ 'install_avg','withdraw_avg' ], dbWhere:'AND scale=5 AND batch_size=100', y_axis:'Latency (ms)'],
     SCPFintentRerouteLatWithFlowObj: [ test:'SCPFintentRerouteLat --params TEST/flowObj=True', table:'intent_reroute_latency_fobj_tests', results:'intent_reroute_latency_fobj_results', file:'IntentRerouteLatDBWithFlowObj', rFile:'SCPFIntentInstallWithdrawRerouteLat.R y', extra:batches, finalResult:0],
-    SCPFscalingMaxIntentsWithFlowObj: [ test:'SCPFscalingMaxIntents --params TEST/flowObj=True', table:'max_intents_fobj_tests', results:'max_intents_fobj_results', file:'ScalingMaxIntentDBWFO', rFile:'SCPFscalingMaxIntents.R y', extra:none, finalResult:0],
+    SCPFscalingMaxIntentsWithFlowObj: [ test:'SCPFscalingMaxIntents --params TEST/flowObj=True', table:'max_intents_fobj_tests', results:'max_intents_fobj_results', file:'ScalingMaxIntentDBWFO', rFile:'SCPFscalingMaxIntents.R y', extra:times, finalResult:0],
     SCPFintentInstallWithdrawLatWithFlowObj: [ test:'SCPFintentInstallWithdrawLat --params TEST/flowObj=True', table:'intent_latency_fobj_tests', results:'intent_latency_fobj_results', file:'IntentInstallWithdrawLatDBWFO', rFile:'SCPFIntentInstallWithdrawRerouteLat.R y', extra:batches, finalResult:0],
     SCPFmastershipFailoverLat: [test:'SCPFmastershipFailoverLat', table:'mastership_failover_tests', results:'mastership_failover_results', file:'mastershipFailoverLatDB', rFile:'SCPFmastershipFailoverLat.R', extra:none, finalResult:1, graphTitle:['Mastership Failover Test'], dbCols:[ 'kill_deact_avg,deact_role_avg' ], dbWhere:'AND scale=5', y_axis:'Latency (ms)' ]
 ]
@@ -47,15 +49,20 @@
     tests[stepName] = SCPFTest(test, toBeRun, prop)
 }
 
+def now = new Date()
 // run the tests
 for ( test in tests.keySet() ){
     tests[test].call()
 }
-
+if( prop["manualRun"] == "false" ){
+    def end = new Date()
+    TimeDuration duration = TimeCategory.minus( end, now )
+    slackSend( color:"#5816EE", message: "SCPF tests ended at: " + end.toString() + "\nTime took : " + duration )
+}
 // The testName should be the key from the SCPF map
 def SCPFTest( testName, toBeRun, prop ) {
     return {
-        catchError{
+        try{
             stage(testName) {
                 if ( toBeRun ){
                     workSpace = "/var/jenkins/workspace/"+testName
@@ -104,30 +111,32 @@
                             ls -al
                             cd '''
                             // Post Results
-                            withCredentials([
-                                string(credentialsId: 'db_pass', variable: 'pass'),
-                                string(credentialsId: 'db_user', variable: 'user'),
-                                string(credentialsId: 'db_host', variable: 'host'),
-                                string(credentialsId: 'db_port', variable: 'port')]) {
-                                    def database_command = pass + "|psql --host=" + host + " --port=" + port + " --username=" + user + " --password --dbname onostest -c \"INSERT INTO " + SCPF[testName]['table'] + " VALUES('\$DATE','" + SCPF[testName]['results'] + "','\$BUILD_NUMBER', '\$ONOSBranch', \$line);\""
-                                    if (testName == "SCPFscaleTopo" || testName == "SCPFswitchLat" || testName == "SCPFportLat") {
-                                        database_command = pass + "|psql --host=" + host + " --port=" + port + " --username=" + user + " --password --dbname onostest -c \"INSERT INTO " + SCPF[testName]['table'] + " VALUES('\$DATE','" + SCPF[testName]['results'] + "','\$BUILD_NUMBER', \$line, '\$ONOSBranch');\""
-                                    }
-                                    sh '''#!/bin/bash
+                            if( prop["manualRun"] == "false" || prop["postResult"] == "true" ){
+                                withCredentials([
+                                    string(credentialsId: 'db_pass', variable: 'pass'),
+                                    string(credentialsId: 'db_user', variable: 'user'),
+                                    string(credentialsId: 'db_host', variable: 'host'),
+                                    string(credentialsId: 'db_port', variable: 'port')]) {
+                                        def database_command = pass + "|psql --host=" + host + " --port=" + port + " --username=" + user + " --password --dbname onostest -c \"INSERT INTO " + SCPF[testName]['table'] + " VALUES('\$DATE','" + SCPF[testName]['results'] + "','\$BUILD_NUMBER', '\$ONOSBranch', \$line);\""
+                                        if (testName == "SCPFscaleTopo" || testName == "SCPFswitchLat" || testName == "SCPFportLat") {
+                                            database_command = pass + "|psql --host=" + host + " --port=" + port + " --username=" + user + " --password --dbname onostest -c \"INSERT INTO " + SCPF[testName]['table'] + " VALUES('\$DATE','" + SCPF[testName]['results'] + "','\$BUILD_NUMBER', \$line, '\$ONOSBranch');\""
+                                        }
+                                        sh '''#!/bin/bash
 
-                                    export DATE=\$(date +%F_%T)
-                                    cd ~
-                                    pwd
-                                    cd /tmp
-                                    while read line
-                                    do
+                                        export DATE=\$(date +%F_%T)
+                                        cd ~
+                                        pwd
+                                        cd /tmp
+                                        while read line
+                                        do
 
-                                    echo \$line
-                                    echo ''' + database_command + '''
+                                        echo \$line
+                                        echo ''' + database_command + '''
 
-                                    done< ''' + SCPF[testName]['file'] + '''
-                                    ''' + getGraphCommand( SCPF[testName]['rFile'], SCPF[testName]['extra'], host, port, user, pass, testName, prop["ONOSBranch"] ) + '''
-                                    ''' + ( SCPF[testName]['finalResult'] ? generateCombinedResultGraph( host,port, user, pass, testName, prop["ONOSBranch"] ) : "" )
+                                        done< ''' + SCPF[testName]['file'] + '''
+                                        ''' + getGraphCommand( SCPF[testName]['rFile'], SCPF[testName]['extra'], host, port, user, pass, testName, prop["ONOSBranch"] ) + '''
+                                        ''' + ( SCPF[testName]['finalResult'] ? generateCombinedResultGraph( host,port, user, pass, testName, prop["ONOSBranch"] ) : "" )
+                                }
                             }
                             // Fetch Logs
                             sh '''#!/bin/bash
@@ -148,7 +157,7 @@
                             fi'''
                         }
                     }
-                    if( prop["manualRun"] == "false" ){
+                    if( prop["manualRun"] == "false" || prop["postResult"] == "true" ){
                         def post = build job: "Pipeline_postjob_BM", propagate: false
                     }
                     node("TestStation-BMs"){
@@ -164,6 +173,12 @@
                     }
                 }
             }
+        }catch (all) {
+            catchError{
+                if( prop["manualRun"] == "false" )
+                    slackSend(color:"FF0000", message: "[" + prop["TestONBranch"] + "]" + testName + " : Failed!\nURL:${BUILD_URL}")
+                Failed
+            }
         }
     }
 }
diff --git a/TestON/JenkinsFile/USECASEJenkinsFile b/TestON/JenkinsFile/USECASEJenkinsFile
index 3c78dbe..e897cfa 100644
--- a/TestON/JenkinsFile/USECASEJenkinsFile
+++ b/TestON/JenkinsFile/USECASEJenkinsFile
@@ -1,4 +1,5 @@
 #!groovy
+import groovy.time.*
 // This is a Jenkinsfile for a scripted pipeline for the USECASETest tests
 
 // TODO: Exception handling around steps
@@ -19,7 +20,7 @@
     "SROnosFailure": [wiki_link:prop["WikiPrefix"]+"-"+"SR Onos node Failure", wiki_file:"SROnosFailureWiki.txt"],
     "SRClusterRestart": [wiki_link:prop["WikiPrefix"]+"-"+"SR Cluster Restart", wiki_file:"SRClusterRestartWiki.txt"],
     "SRDynamic": [wiki_link:prop["WikiPrefix"]+"-"+"SR Dynamic Config", wiki_file:"SRDynamicWiki.txt"],
-    "SRHA": [wiki_link:prop["WikiPrefix"]+"-"+"SR High Availability", wiki_file:"SRHighAvailabilityWiki.txt"],
+    "SRHighAvailability": [wiki_link:prop["WikiPrefix"]+"-"+"SR High Availability", wiki_file:"SRHighAvailabilityWiki.txt"],
     "USECASE_SdnipFunction": [wiki_link:prop["WikiPrefix"]+"-"+"SDNIP Function", wiki_file:"USECASE_SdnipFunctionWiki.txt"],
     "USECASE_SdnipFunctionCluster": [wiki_link:prop["WikiPrefix"]+"-"+"SDNIP Function Cluster", wiki_file:"USECASE_SdnipFunctionClusterWiki.txt"]
 ]
@@ -42,15 +43,20 @@
     tests[stepName] = USECASETest(test, toBeRun, prop)
 }
 
+def now = new Date()
 // run the tests
 for ( test in tests.keySet() ){
     tests[test].call()
 }
-
+if( prop["manualRun"] == "false" ){
+    def end = new Date()
+    TimeDuration duration = TimeCategory.minus( end, now )
+    slackSend( color:"#5816EE", message: "USECASE tests ended at: " + end.toString() + "\nTime took : " + duration )
+}
 // The testName should be the key from the FUNC
 def USECASETest( testName, toBeRun, prop ) {
     return {
-        catchError{
+        try{
             stage(testName) {
                 if ( toBeRun ){
                     workSpace = "/var/jenkins/workspace/"+testName
@@ -119,7 +125,7 @@
                             ls -al
                             cd '''
 
-                            if( prop["manualRun"] == "false" ){
+                            if( prop["manualRun"] == "false" || prop["postResult"] == "true"  ){
                                 // Post Results
                                 withCredentials([
                                     string(credentialsId: 'db_pass', variable: 'pass'),
@@ -164,7 +170,7 @@
 
                         }
                     }
-                    if( prop["manualRun"] == "false" ){
+                    if( prop["manualRun"] == "false" || prop["postResult"] == "true" ){
                         def post = build job: "Pipeline_postjob_BM", propagate: false,
                                     parameters: [
                                         string(name: 'Wiki_Contents', value: fileContents),
@@ -184,6 +190,12 @@
                     }
                 }
             }
+        }catch (all) {
+            catchError{
+                if( prop["manualRun"] == "false" )
+                    slackSend(color:"FF0000", message: "[" + prop["TestONBranch"] + "]" + testName + " : Failed!\nURL:${BUILD_URL}")
+                Failed
+            }
         }
     }
 }
\ No newline at end of file