Merge "Fix a bug of expect result for timeout"
diff --git a/TestON/JenkinsFile/FabricJenkinsfileTrigger b/TestON/JenkinsFile/FabricJenkinsfileTrigger
index f5a7dc2..7f62fe4 100644
--- a/TestON/JenkinsFile/FabricJenkinsfileTrigger
+++ b/TestON/JenkinsFile/FabricJenkinsfileTrigger
@@ -31,7 +31,7 @@
 // set the versions of the onos.
 current_version = "master"
 previous_version = "1.15"
-before_previous_version = "1.14"
+before_previous_version = "2.0"
 
 // Function that will initialize the configuration of the Fabric.
 funcs.initializeTrend( "Fabric" )
diff --git a/TestON/JenkinsFile/README b/TestON/JenkinsFile/README
index 1e837e4..73c9749 100644
--- a/TestON/JenkinsFile/README
+++ b/TestON/JenkinsFile/README
@@ -12,7 +12,7 @@
   2) change the portion on line 31 :
      current_version = "master"
      previous_version = "1.15"
-     before_previous_version = "1.14"
+     before_previous_version = "2.0"
 
 
 2. If you want to change the branch to be run for the Segment Routing tests :
@@ -21,18 +21,18 @@
   2) change the variable values on line 32:
      current_version = "master"
      previous_version = "1.15"
-     before_previous_version = "1.14"
+     before_previous_version = "2.0"
   3) If you want certain branch to be run on different machine then change
      a) line 155 - 157:
         testcases[ "SR" ][ "nodeName" ][ <node you want> ] 0: Fabric2, 1: Fabric3, 2: Fabric4
      b) Then, go to dependencies/JenkinsCommonFuncs.groovy
         change def fabricOn( branch ) on line 86.
-        Currently, master, 1.15 and 1.14 run on Fabric4, 2 and 3 respectively.
-     c) Make sure to change the URLs on Wiki as well. If 1.14 runs on Fabric2 instead of Fabric3, then you
+        Currently, master, 2.0 and 1.15 run on Fabric4, 3 and 2 respectively.
+     c) Make sure to change the URLs on Wiki as well. If 1.15 runs on Fabric2 instead of Fabric3, then you
         have to update the URL from
-          https://jenkins.onosproject.org/view/QA/job/postjob-Fabric3/lastSuccessfulBuild/artifact/SRBridging_onos-1.14_20-builds_graph.jpg
+          https://jenkins.onosproject.org/view/QA/job/postjob-Fabric3/lastSuccessfulBuild/artifact/SRBridging_onos-1.15_20-builds_graph.jpg
           to
-          https://jenkins.onosproject.org/view/QA/job/postjob-Fabric2/lastSuccessfulBuild/artifact/SRBridging_onos-1.14_20-builds_graph.jpg
+          https://jenkins.onosproject.org/view/QA/job/postjob-Fabric2/lastSuccessfulBuild/artifact/SRBridging_onos-1.15_20-builds_graph.jpg
 
 
 3. If you want to upload additional logs for the Segment Routing tests:
diff --git a/TestON/JenkinsFile/SRHAJenkinsFile b/TestON/JenkinsFile/SRHAJenkinsFile
index b13290f..98021fd 100644
--- a/TestON/JenkinsFile/SRHAJenkinsFile
+++ b/TestON/JenkinsFile/SRHAJenkinsFile
@@ -36,7 +36,7 @@
 jobName = env.JOB_NAME
 
 // read the TestON.property depends on which branch it is running.
-// ( currently master on Fabric4, 1.15 on Fabric2 and 1.14 on Fabric3 )
+// ( currently master on Fabric4, 2.0 on Fabric3 and 1.15 on Fabric2 )
 def prop = null
 prop = funcs.getProperties()
 SRHA = test_lists.getAllTheTests( prop[ "WikiPrefix" ] )[ "SRHA" ]
diff --git a/TestON/JenkinsFile/SRJenkinsFile b/TestON/JenkinsFile/SRJenkinsFile
index 30fba24..9b85b89 100644
--- a/TestON/JenkinsFile/SRJenkinsFile
+++ b/TestON/JenkinsFile/SRJenkinsFile
@@ -36,7 +36,7 @@
 funcs.additionalInitForSR( jobName )
 
 // read the TestON.property depends on which branch it is running.
-// ( currently master on Fabric4, 1.15 on Fabric2 and 1.14 on Fabric3 )
+// ( currently master on Fabric4, 1.15 on Fabric2 and 2.0 on Fabric3 )
 def prop = null
 prop = funcs.getProperties()
 SR = test_lists.getAllTheTests( prop[ "WikiPrefix" ] )[ "SR" ]
diff --git a/TestON/JenkinsFile/VM_BMJenkinsfileTrigger b/TestON/JenkinsFile/VM_BMJenkinsfileTrigger
index f2e8bc9..a1f7f29 100644
--- a/TestON/JenkinsFile/VM_BMJenkinsfileTrigger
+++ b/TestON/JenkinsFile/VM_BMJenkinsfileTrigger
@@ -30,7 +30,7 @@
 fileRelated.init()
 current_version = "master"
 previous_version = "1.15"
-before_previous_version = "1.14"
+before_previous_version = "2.0"
 
 // init trend graphs to be on VM.
 funcs.initializeTrend( "VM" );
diff --git a/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy b/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
index 0a37eac..d309f44 100644
--- a/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
+++ b/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
@@ -83,7 +83,7 @@
 
 def additionalInitForSR( branch ){
     // additional setup for SegmentRouting tests to determine the machine depends on the branch it is running.
-    // branch : branch of the onos. ( master, 1.15, 1.14... )
+    // branch : branch of the onos. ( master, 2.0, 1.15... )
 
     testMachine = ( ( new StringBuilder( testMachine ) ).insert( testMachine.size() - 1, fabricOn( branch ) ) ).
             toString()
@@ -98,14 +98,15 @@
 
 def fabricOn( branch ){
     // gets the fabric machines with the branch of onos.
-    // branch : master, 1.15, 1.14...
+    // branch : master, 2.0, 1.15...
     // branch.reverse().take(4).reverse() will get last 4 characters of the string.
-    switch ( branch.reverse().take( 4 ).reverse() ){
-        case "ster": return "4"
-        case "1.15": return "2"
-        case "1.14": return "3"
-        case "1.13": return "2"
-        case "1.12": return "3"
+    switch ( branch.reverse().take( 3 ).reverse() ){
+        case "ter": return "4"
+        case "2.0": return "3"
+        case ".15": return "2"
+        case ".14": return "3"
+        case ".13": return "2"
+        case ".12": return "3"
         default: return "4"
     }
 }
@@ -294,7 +295,7 @@
 def postLogs( testName, prefix ){
     // posting logs of the onos jobs specifically SR tests
     // testName : name of the test
-    // prefix : branch prefix ( master, 1.15, 1.14 ... )
+    // prefix : branch prefix ( master, 2.0, 1.15 ... )
 
     resultURL = ""
     if ( testType == "SR" ){
@@ -384,7 +385,7 @@
                     def fileContents = ""
                     node( testMachine ) {
                         withEnv( [ 'ONOSBranch=' + prop[ "ONOSBranch" ],
-                                   'ONOSJVMHeap=' + prop[ "ONOSJVMHeap" ],
+                                   'ONOSJAVAOPTS=' + prop[ "ONOSJAVAOPTS" ],
                                    'TestONBranch=' + prop[ "TestONBranch" ],
                                    'ONOSTag=' + prop[ "ONOSTag" ],
                                    'WikiPrefix=' + prop[ "WikiPrefix" ],
@@ -463,7 +464,7 @@
     // stat_file : file name with full path for Rscript for the stat graph
     // pie_file : file name with full path for Rscript for the pie graph
     // type : type of the test ( USECASE, FUNC, HA )
-    // branch : branch of the test ( master, onos-1.15, onos-1.14 )
+    // branch : branch of the test ( master, onos-2.0, onos-1.15 )
     // testListPart : list of the test to be included
     // save_path : path that will save the graphs to
     // pieTestListPart : list of the test for pie graph
@@ -587,9 +588,11 @@
     else if ( branch == "1.13" )
         return "onos-1.13"
     else if ( branch == "1.14" )
-        return "onos-1.14"
+        return "onos-1.15"
     else if ( branch == "1.15" )
-        return "onos-1.14"
+        return "onos-1.15"
+    else if ( branch == "2.0" )
+        return "master"
     else
         return "master"
 }
diff --git a/TestON/JenkinsFile/dependencies/JenkinsTestONTests.groovy b/TestON/JenkinsFile/dependencies/JenkinsTestONTests.groovy
index 1f9df18..e1f1495 100644
--- a/TestON/JenkinsFile/dependencies/JenkinsTestONTests.groovy
+++ b/TestON/JenkinsFile/dependencies/JenkinsTestONTests.groovy
@@ -26,7 +26,7 @@
 
 def getAllTheTests( wikiPrefix ){
     // This contains the dictionary of the test and the category of them
-    // wikiPrefix : master, 1.15, 1.14 ...
+    // wikiPrefix : master, 2.0, 1.15 ...
 
     // category: it will be used to distinguish which category to be run on which days ( basic,extra_A, extra_B ... )
     // day: it will be used to display the schedule of the test to be run to the slack. It is empty in the first place but will be
diff --git a/TestON/JenkinsFile/dependencies/TriggerFuncs.groovy b/TestON/JenkinsFile/dependencies/TriggerFuncs.groovy
index 2ca4f72..effc766 100644
--- a/TestON/JenkinsFile/dependencies/TriggerFuncs.groovy
+++ b/TestON/JenkinsFile/dependencies/TriggerFuncs.groovy
@@ -135,7 +135,7 @@
             echo "TestONBranch=''' + test_branch + '''" >> /var/jenkins/TestONOS.property
             echo "ONOSTag=''' + onosTag + '''" >> /var/jenkins/TestONOS.property
             echo "WikiPrefix=''' + wiki + '''" >> /var/jenkins/TestONOS.property
-            echo "ONOSJVMHeap=''' + env.ONOSJVMHeap + '''" >> /var/jenkins/TestONOS.property
+            echo "ONOSJAVAOPTS=''' + env.ONOSJAVAOPTS + '''" >> /var/jenkins/TestONOS.property
             echo "Tests=''' + tests + '''" >> /var/jenkins/TestONOS.property
             echo "postResult=''' + postResult + '''" >> /var/jenkins/TestONOS.property
             echo "manualRun=''' + manually_run + '''" >> /var/jenkins/TestONOS.property
@@ -213,8 +213,8 @@
         git branch
         git log -1 --decorate
         echo -e "\n##### set jvm heap size to 8G #####"
-        echo ${ONOSJVMHeap}
-        inserted_line="export JAVA_OPTS=\"\${ONOSJVMHeap}\""
+        echo ${ONOSJAVAOPTS}
+        inserted_line="export JAVA_OPTS=\"\${ONOSJAVAOPTS}\""
         sed -i "s/bash/bash\\n$inserted_line/" ~/onos/tools/package/bin/onos-service
         echo "##### Check onos-service setting..... #####"
         cat ~/onos/tools/package/bin/onos-service
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index b87e344..b09fa70 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -522,7 +522,7 @@
             else:
                 main.cleanAndExit()
 
-    def sendline( self, cmdStr, showResponse=False, debug=False, timeout=10, noExit=False, relaxedRegex=True ):
+    def sendline( self, cmdStr, showResponse=False, debug=False, timeout=10, noExit=False, relaxedRegex=True, expectJson=False ):
         """
         A wrapper around pexpect's sendline/expect. Will return all the output from a given command
 
@@ -558,7 +558,7 @@
                 main.log.debug( self.name + ": " + repr( response ) )
 
             # Remove control codes from karaf 4.2.1
-            karafEscape = re.compile( r"('(0|1)~\'|\r\r\r\n\x1b\[A\x1b\[79Cx|\x1b(>|=)|\x1b\[90m~)" )
+            karafEscape = re.compile( r"('(0|1)~\'|\r\r\r\n\x1b\[A\x1b\[79C(x|\s)?|\x1b(>|=)|\x1b\[90m~)" )
             response = karafEscape.sub( '', response )
             if debug:
                 main.log.debug( self.name + ": karafEscape output" )
@@ -571,6 +571,8 @@
                 main.log.debug( self.name + ": " + repr( response ) )
 
             # Remove ANSI color control strings from output
+            # NOTE: karaf is sometimes adding a single character then two
+            #       backspaces and sometimes adding 2 characters with 2 backspaces??
             backspaceEscape = re.compile( r'((..\x08\x08)|(.|\s)\x08)' )
             response = backspaceEscape.sub( '', response )
             if debug:
@@ -591,20 +593,40 @@
                 main.log.debug( self.name + ": " + repr( response ) )
 
             # parse for just the output, remove the cmd from response
+            cmdPattern = cmdStr.strip()
+            main.log.debug( "REGEX PATTERN SET TO: " + repr( cmdPattern ) +
+                            "\nSENT COMMAND STRING WAS: " + repr( cmdStr ) )
             if relaxedRegex:
+                cmdPattern = cmdPattern.split( '|' )[ -1 ].strip()
+                main.log.debug( "REGEX PATTERN SET TO: " + repr( cmdPattern ) +
+                                "\nSENT COMMAND STRING WAS: " + repr( cmdStr ) )
                 # This was added because karaf 4.2 is stripping some characters from the command echo
-                endStr = cmdStr.split( '|' )[-1]
-                output = response.split( endStr.strip(), 1 )
-            else:
-                output = response.split( endStr.strip(), 1 )
+                output = response.split( cmdPattern, 1 )
+                if expectJson:
+                    main.log.warn( "Relaxed Regex: Searching for a json string amongst the output" )
+                    jsonPattern = r'\{.*\}'
+                    match = re.search( jsonPattern, output[ 0 ] )
+                    if match:
+                        output = [ '' , match.group( 0 ) ]  # We expect a list with the second element to be the output
                 if len( output ) < 2:
                     main.log.warn( "Relaxing regex match to last 5 characters of the sent command" )
-                    output = response.split( endStr.strip()[-5:], 1 )
+                    cmdPattern = cmdPattern[ -5: ]
+                    main.log.debug( "REGEX PATTERN SET TO: " + repr( cmdPattern ) +
+                                    "\nSENT COMMAND STRING WAS: " + repr( cmdStr ) )
+                    output = response.split( cmdPattern, 1 )
+            else:
+                output = response.split( cmdPattern, 1 )
+                if len( output ) < 2:  # TODO: Should we do this without the relaxedRegex flag?
+                    main.log.warn( "Relaxing regex match to last 5 characters of the sent command" )
+                    output = response.split( cmdPattern[ -5: ], 1 )
             if output:
                 if debug:
                     main.log.debug( self.name + ": split output" )
                     for r in output:
                         main.log.debug( self.name + ": " + repr( r ) )
+                if len( output ) == 1:
+                    main.log.error( "Could not remove sent command echo from output" )
+                    return output
                 output = output[ 1 ].strip()
             if showResponse:
                 main.log.info( "Response from ONOS: {}".format( output ) )
@@ -666,7 +688,7 @@
             return parsed.group( 1 )
         except IndexError:
             main.log.exception( self.name + ": Object not as expected" )
-            main.log.debug( "response: {}".format( repr( response ) ) )
+            main.log.debug( "response: {}".format( repr( numLines ) ) )
             return None
         except TypeError:
             main.log.exception( self.name + ": Object not as expected" )
@@ -2799,6 +2821,10 @@
                 return None
             return totalFlows
 
+        except IndexError:
+            main.log.exception( "{}: Object not as expected!".format( self.name ) )
+            main.log.debug( "rawFlows: {}".format( rawFlows ) )
+            return None
         except ( TypeError, ValueError ):
             main.log.exception( "{}: Object not as expected!".format( self.name ) )
             return None
@@ -4210,9 +4236,11 @@
                     componentStr += " " + propName
             if jsonFormat:
                 baseStr += " -j"
+                expectJson = True
             elif short:
                 baseStr += " -s"
-            output = self.sendline( baseStr + cmdStr + componentStr )
+                expectJson = False
+            output = self.sendline( baseStr + cmdStr + componentStr, expectJson=expectJson )
             assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
@@ -4574,6 +4602,9 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            main.log.debug( self.name + ": Counters unparsed: " + output )
+            output = output.split( "\r\n" )[ -1 ]
+            main.log.debug( self.name + ": Counters parsed: " + output )
             assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
@@ -4681,6 +4712,9 @@
             cmdStr = "value-test {} {}".format( valueName,
                                                 operation )
             output = self.distPrimitivesSend( cmdStr )
+            main.log.debug( self.name + ": value test unparsed: " + output )
+            output = output.split( "\r\n" )[ -1 ]
+            main.log.debug( self.name + ": value test parsed: " + output )
             pattern = "(\w+)"
             match = re.search( pattern, output )
             if match:
@@ -5401,8 +5435,8 @@
             elif mode == 'first':
                 cmd = cmd + " | grep \'" + searchTerm + "\'" + " | head -n 1"
             elif mode == 'num':
-                cmd = cmd + " | grep -c \'" + searchTerm + "\'"
-                num = self.sendline( cmd )
+                cmd = cmd + " | grep \'" + searchTerm + "\' | wc -l"
+                num = self.lineCount( cmd )
                 return num
             elif mode == 'total':
                 totalLines = self.lineCount( "cat /opt/onos/log/karaf.log | wc -l" )
@@ -5945,6 +5979,9 @@
             operation = "totalPending"
             cmdStr = " ".join( [ prefix, queueName, operation ] )
             output = self.distPrimitivesSend( cmdStr )
+            main.log.debug( self.name + ": work queue unparsed: " + output )
+            output = output.split( "\r\n" )[ -1 ]
+            main.log.debug( self.name + ": work queue parsed: " + output )
             pattern = r'\d+'
             if "Invalid operation name" in output:
                 main.log.warn( output )
@@ -5978,6 +6015,9 @@
             operation = "totalCompleted"
             cmdStr = " ".join( [ prefix, queueName, operation ] )
             output = self.distPrimitivesSend( cmdStr )
+            main.log.debug( self.name + ": work queue unparsed: " + output )
+            output = output.split( "\r\n" )[ -1 ]
+            main.log.debug( self.name + ": work queue parsed: " + output )
             pattern = r'\d+'
             if "Invalid operation name" in output:
                 main.log.warn( output )
@@ -6011,6 +6051,9 @@
             operation = "totalInProgress"
             cmdStr = " ".join( [ prefix, queueName, operation ] )
             output = self.distPrimitivesSend( cmdStr )
+            main.log.debug( self.name + ": work queue unparsed: " + output )
+            output = output.split( "\r\n" )[ -1 ]
+            main.log.debug( self.name + ": work queue parsed: " + output )
             pattern = r'\d+'
             if "Invalid operation name" in output:
                 main.log.warn( output )
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 2b17d66..e47d34d 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -1435,6 +1435,8 @@
             i = self.handle.expect( [ self.prompt, pexpect.TIMEOUT, "Password:" ], timeout )
             if i == 0:
                 main.log.info( self.name + ": " + node + " is up" )
+                # FIXME: for now we sleep 5s for CLI to become ready
+                time.sleep( 5 )
                 return main.TRUE
             elif i == 1 or i == 2:
                 # NOTE: since this function won't return until ONOS is ready,
diff --git a/TestON/tests/FUNC/FUNCintent/FUNCintent.py b/TestON/tests/FUNC/FUNCintent/FUNCintent.py
index 86b9f53..6c12a4f 100644
--- a/TestON/tests/FUNC/FUNCintent/FUNCintent.py
+++ b/TestON/tests/FUNC/FUNCintent/FUNCintent.py
@@ -897,41 +897,6 @@
                                  onpass=main.assertReturnString,
                                  onfail=main.assertReturnString )
 
-        main.step( "Protected: Add point intents between h1 and h9" )
-        main.assertReturnString = "Assertion Result for protected point intent\n"
-        senders = [
-            { "name": "h1", "device": "of:0000000000000005/1", "mac": "00:00:00:00:00:01" }
-        ]
-        recipients = [
-            { "name": "h9", "device": "of:0000000000000006/1", "mac": "00:00:00:00:00:09" }
-        ]
-        testResult = main.FALSE
-        installResult = main.intents.installPointIntent(
-            main,
-            name="Protected",
-            senders=senders,
-            recipients=recipients,
-            protected=True )
-
-        if installResult:
-            testResult = main.intents.testPointIntent(
-                main,
-                name="Protected",
-                intentId=installResult,
-                senders=senders,
-                recipients=recipients,
-                sw1="s5",
-                sw2="s2",
-                protected=True,
-                expectedLink=18 )
-        else:
-            main.intents.removeAllExistIntents( main )
-
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=testResult,
-                                 onpass=main.assertReturnString,
-                                 onfail=main.assertReturnString )
-
         main.step( "IPV4_2: Add point intents between h1 and h9" )
         main.assertReturnString = "Assertion Result for IPV4 no mac address point intents\n"
         senders = [
diff --git a/TestON/tests/HA/HAbackupRecover/HAbackupRecover.py b/TestON/tests/HA/HAbackupRecover/HAbackupRecover.py
index 7010f93..e6ba11e 100644
--- a/TestON/tests/HA/HAbackupRecover/HAbackupRecover.py
+++ b/TestON/tests/HA/HAbackupRecover/HAbackupRecover.py
@@ -88,14 +88,15 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
+        applyFuncs = [ main.HA.removeKarafConsoleLogging ]
+
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
-                applyFuncs = None
             else:
-                applyFuncs = main.HA.startingMininet
+                applyFuncs.append( main.HA.startingMininet )
         except (KeyError, IndexError):
-            applyFuncs = main.HA.startingMininet
+            applyFuncs.append( main.HA.startingMininet )
 
         main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, extraApply=applyFuncs )
 
diff --git a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
index 14a0898..35800d2 100644
--- a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
@@ -88,14 +88,15 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
+        applyFuncs = [ main.HA.removeKarafConsoleLogging ]
+
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
-                applyFuncs = None
             else:
-                applyFuncs = main.HA.startingMininet
+                applyFuncs.append( main.HA.startingMininet )
         except (KeyError, IndexError):
-            applyFuncs = main.HA.startingMininet
+            applyFuncs.append( main.HA.startingMininet )
 
         main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, extraApply=applyFuncs )
 
diff --git a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
index 277e098..d770e83 100644
--- a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
+++ b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
@@ -91,8 +91,8 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.customizeOnosGenPartitions, main.HA.copyBackupConfig ]
-        applyArgs = [ None, None ]
+        applyFuncs = [ main.HA.removeKarafConsoleLogging, main.HA.customizeOnosGenPartitions, main.HA.copyBackupConfig ]
+        applyArgs = [ None, None, None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
index e3daf8e..9020edc 100644
--- a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
+++ b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
@@ -90,8 +90,8 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.customizeOnosGenPartitions ]
-        applyArgs = [ None ]
+        applyFuncs = [ main.HA.removeKarafConsoleLogging, main.HA.customizeOnosGenPartitions ]
+        applyArgs = [ None, None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/HAkillNodes/HAkillNodes.py b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
index 560b785..1be271f 100644
--- a/TestON/tests/HA/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
@@ -88,10 +88,11 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.customizeOnosGenPartitions,
+        applyFuncs = [ main.HA.removeKarafConsoleLogging,
+                       main.HA.customizeOnosGenPartitions,
                        main.HA.copyBackupConfig,
                        main.ONOSbench.preventAutoRespawn ]
-        applyArgs = [ None, None, None ]
+        applyArgs = [ None, None, None, None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/HApowerFailure/HApowerFailure.py b/TestON/tests/HA/HApowerFailure/HApowerFailure.py
index 85801e0..aa04792 100644
--- a/TestON/tests/HA/HApowerFailure/HApowerFailure.py
+++ b/TestON/tests/HA/HApowerFailure/HApowerFailure.py
@@ -88,10 +88,11 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.customizeOnosGenPartitions,
+        applyFuncs = [ main.HA.removeKarafConsoleLogging,
+                       main.HA.customizeOnosGenPartitions,
                        main.HA.copyBackupConfig,
                        main.ONOSbench.preventAutoRespawn ]
-        applyArgs = [ None, None, None ]
+        applyArgs = [ None, None, None, None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/HAsanity/HAsanity.py b/TestON/tests/HA/HAsanity/HAsanity.py
index b6d00c0..eecd0f1 100644
--- a/TestON/tests/HA/HAsanity/HAsanity.py
+++ b/TestON/tests/HA/HAsanity/HAsanity.py
@@ -87,14 +87,15 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
+        applyFuncs = [ main.HA.removeKarafConsoleLogging ]
+
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
-                applyFuncs = None
             else:
-                applyFuncs = main.HA.startingMininet
+                applyFuncs.append( main.HA.startingMininet )
         except (KeyError, IndexError):
-            applyFuncs = main.HA.startingMininet
+            applyFuncs.append( main.HA.startingMininet )
 
         main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs, stopAtomix=True,
diff --git a/TestON/tests/HA/HAscaling/HAscaling.py b/TestON/tests/HA/HAscaling/HAscaling.py
index 6929ae7..19b0ec3 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.py
+++ b/TestON/tests/HA/HAscaling/HAscaling.py
@@ -97,8 +97,8 @@
         main.log.debug( scale )
         main.Cluster.setRunningNode( int( re.search( "\d+", scale ).group( 0 ) ) )
 
-        applyFuncs = []
-        applyArgs = []
+        applyFuncs = [ main.HA.removeKarafConsoleLogging ]
+        applyArgs = [ None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index af383f1..c23284b 100644
--- a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -92,9 +92,9 @@
         main.testSetUp.evnSetupConclusion( stepResult )
 
         cellApps = str( main.params["ENV"]["appString"] )
-        cellNAme = str( main.params["ENV"]["appString"] )
-        applyFuncs = [ main.testSetUp.createApplyCell ]
-        applyArgs = [ [ main.Cluster, True, cellName , cellApps, "", True, main.Cluster.runningNodes[ 0 ].ipAddress ] ]
+        cellName = str( main.params["ENV"]["appString"] )
+        applyFuncs = [ main.HA.removeKarafConsoleLogging, main.testSetUp.createApplyCell ]
+        applyArgs = [ None, [ main.Cluster, True, cellName , cellApps, "", True, main.Cluster.runningNodes[ 0 ].ipAddress ] ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.py b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
index 998f29c..0c6c7d0 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
@@ -88,8 +88,8 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.customizeOnosGenPartitions, main.HA.copyBackupConfig ]
-        applyArgs = [ None, None ]
+        applyFuncs = [ main.HA.removeKarafConsoleLogging, main.HA.customizeOnosGenPartitions, main.HA.copyBackupConfig ]
+        applyArgs = [ None, None, None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.py b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
index 52da8ed..e2efe92 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.py
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
@@ -89,8 +89,8 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.swapNodeMetadata ]
-        applyArgs = [ None ]
+        applyFuncs = [ main.HA.removeKarafConsoleLogging, main.HA.swapNodeMetadata ]
+        applyArgs = [ None, None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.py b/TestON/tests/HA/HAupgrade/HAupgrade.py
index f71cf79..cea03c8 100644
--- a/TestON/tests/HA/HAupgrade/HAupgrade.py
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.py
@@ -92,8 +92,8 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.copyBackupConfig ]
-        applyArgs = [ None ]
+        applyFuncs = [ main.HA.removeKarafConsoleLogging, main.HA.copyBackupConfig ]
+        applyArgs = [ None, None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
index db5e503..f5ae894 100644
--- a/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
+++ b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
@@ -92,8 +92,8 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.copyBackupConfig ]
-        applyArgs = [ None ]
+        applyFuncs = [ main.HA.removeKarafConsoleLogging, main.HA.copyBackupConfig ]
+        applyArgs = [ None, None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
diff --git a/TestON/tests/HA/dependencies/HA.py b/TestON/tests/HA/dependencies/HA.py
index 4342f96..c809ffe 100644
--- a/TestON/tests/HA/dependencies/HA.py
+++ b/TestON/tests/HA/dependencies/HA.py
@@ -30,6 +30,12 @@
         self.default = ''
         main.topoMappings = {}
 
+    def removeKarafConsoleLogging( self ):
+        main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
+        main.ONOSbench.handle.expect( main.ONOSbench.prompt )
+        main.ONOSbench.handle.sendline( "sed -i 's/-Dkaraf.log.console=INFO //g' tools/package/bin/onos-service" )
+        main.ONOSbench.handle.expect( main.ONOSbench.prompt )
+
     def customizeOnosGenPartitions( self ):
         # copy gen-partions file to ONOS
         # NOTE: this assumes TestON and ONOS are on the same machine
diff --git a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py
index 8b798bd..f4c1c8e 100644
--- a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py
+++ b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py
@@ -202,7 +202,7 @@
                                 continue
 
                         try:
-                            latency = int( installResult.split()[ 5 ] )
+                            latency = int( installResult.split()[ -2 ] )
                             main.log.info( installResult )
                         except:
                             main.log.error( "Failed to get latency, ignore this iteration." )
@@ -245,7 +245,7 @@
                                 continue
 
                         try:
-                            latency = int( withdrawResult.split()[ 5 ] )
+                            latency = int( withdrawResult.split()[ -2 ] )
                             main.log.info( withdrawResult )
                         except:
                             main.log.error( "Failed to get latency, ignore this iteration." )
diff --git a/TestON/tests/SCPF/SCPFmastershipFailoverLat/SCPFmastershipFailoverLat.params b/TestON/tests/SCPF/SCPFmastershipFailoverLat/SCPFmastershipFailoverLat.params
index 497bb9d..fa17b67 100644
--- a/TestON/tests/SCPF/SCPFmastershipFailoverLat/SCPFmastershipFailoverLat.params
+++ b/TestON/tests/SCPF/SCPFmastershipFailoverLat/SCPFmastershipFailoverLat.params
@@ -53,7 +53,7 @@
 
         # Percent of data that can be invalid per sample size iteration.
         # Any more than this will cause the test to fail and stop. Must be a float between 0 and 1.
-        <failPercent>0.25</failPercent>
+        <failPercent>0.5</failPercent>
 
    </TEST>
 
diff --git a/TestON/tests/SCPF/SCPFmastershipFailoverLat/SCPFmastershipFailoverLat.py b/TestON/tests/SCPF/SCPFmastershipFailoverLat/SCPFmastershipFailoverLat.py
index 700008b..b0dd227 100644
--- a/TestON/tests/SCPF/SCPFmastershipFailoverLat/SCPFmastershipFailoverLat.py
+++ b/TestON/tests/SCPF/SCPFmastershipFailoverLat/SCPFmastershipFailoverLat.py
@@ -220,7 +220,7 @@
                 main.log.warn( "Latency was NOT obtained from tshark successfully." )
 
             validDataCheck = False
-            if tsharkLatCheck:
+            if eventLatCheck and tsharkLatCheck:
                 main.log.info( "instanceDeactivated: " + str( instanceDeactivated ) )
                 main.log.info( "masterChanged: " + str( masterChanged ) )
                 main.log.info( "roleRequestLat: " + str( roleRequestLat ) )
@@ -271,8 +271,8 @@
             main.log.info( "Checking ONOS nodes." )
             nodeResults = utilities.retry( main.Cluster.nodesCheck,
                                            False,
-                                           sleep=1,
-                                           attempts=3 )
+                                           sleep=5,
+                                           attempts=50 )
 
             if not nodeResults:
                 main.log.error( "Nodes check NOT successful." )
diff --git a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
index 39f73bf..fee4291 100755
--- a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
+++ b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
@@ -78,7 +78,7 @@
 
     <TOPOLOGY>
         <topology>torus</topology>
-        <scale>10,20,30,40,50,55,60</scale>
+        <scale>10,20,30,40,50</scale>
         <host>True</host>
     </TOPOLOGY>
 
diff --git a/TestON/tests/SCPF/SCPFscaleTopo/dependencies/scaleTopoFunction.py b/TestON/tests/SCPF/SCPFscaleTopo/dependencies/scaleTopoFunction.py
index b59a833..c97006c 100644
--- a/TestON/tests/SCPF/SCPFscaleTopo/dependencies/scaleTopoFunction.py
+++ b/TestON/tests/SCPF/SCPFscaleTopo/dependencies/scaleTopoFunction.py
@@ -39,7 +39,7 @@
         timeString = timeString[ 0 ]
         from datetime import datetime
         # convert time string to timestamp
-        t = datetime.strptime( timeString, "%Y-%m-%d %H:%M:%S,%f" )
+        t = datetime.strptime( timeString, "%Y-%m-%dT%H:%M:%S,%f" )
         import time
         timestamp = time.mktime( t.timetuple() )
         timestamp += int( t.microsecond / 1000 ) / 1000.0
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
index 72d8925..bfcf49f 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
@@ -39,7 +39,7 @@
         <OnosDiscovery>45</OnosDiscovery>
         <loadNetcfgSleep>5</loadNetcfgSleep>
         <startMininetSleep>25</startMininetSleep>
-        <dhcpSleep>60</dhcpSleep>
+        <dhcpSleep>120</dhcpSleep>
         <balanceMasterSleep>10</balanceMasterSleep>
     </timers>
 
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 0b32c5a..e5433f9 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -321,13 +321,8 @@
         """
         Adds an ONOS static route with the use route-add command.
         """
-        main.step("Add static route for subnet {0} towards router interface {1}".format(subnet, intf))
         routeResult = main.Cluster.active( 0 ).addStaticRoute(subnet, intf)
 
-        utilities.assert_equals( expect=True, actual=( not routeResult ),
-                                 onpass="route-add command succeeded",
-                                 onfail="route-add command failed")
-
     @staticmethod
     def checkGroupsForBuckets( main, deviceId, subnetDict, routingTable=30 ):
         """
@@ -898,7 +893,7 @@
                 main.log.debug( "{} components not ACTIVE: \n{}".format(
                     ctrl.name,
                     ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to kill ONOS, stopping test" )
+            main.log.error( "Failed to verify nodes, stopping test" )
             main.cleanAndExit()
 
     @staticmethod