Merge "ONOS-2892 More robust intent/flow checking"
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index cf2f54a..2eca3da 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1990,31 +1990,44 @@
             main.cleanup()
             main.exit()
 
-    def checkFlowsState( self ):
+    def checkFlowsState( self, isPENDING_ADD = True ):
         """
         Description:
             Check the if all the current flows are in ADDED state or
             PENDING_ADD state
+        Optional:
+            * isPENDING_ADD: whether the PENDING_ADD is also a correct status
         Return:
             returnValue - Returns main.TRUE only if all flows are in
-                          ADDED state or PENDING_ADD, return main.FALSE
-                          otherwise.
+                          ADDED state or PENDING_ADD if the PENDING_ADD
+                          parameter is set true, return main.FALSE otherwise.
         """
         try:
             tempFlows = json.loads( self.flows() )
             #print tempFlows[0]
             returnValue = main.TRUE
 
-            for device in tempFlows:
-                for flow in device.get( 'flows' ):
-                    if flow.get( 'state' ) != 'ADDED' and flow.get( 'state' ) != \
-                            'PENDING_ADD':
+            if isPENDING_ADD:
+                for device in tempFlows:
+                    for flow in device.get( 'flows' ):
+                        if flow.get( 'state' ) != 'ADDED' and \
+                            flow.get( 'state' ) != 'PENDING_ADD':
 
-                        main.log.info( self.name + ": flow Id: " +
-                                       str( flow.get( 'groupId' ) ) +
-                                       " | state:" +
-                                       str( flow.get( 'state' ) ) )
-                        returnValue = main.FALSE
+                            main.log.info( self.name + ": flow Id: " +
+                                           str( flow.get( 'groupId' ) ) +
+                                           " | state:" +
+                                           str( flow.get( 'state' ) ) )
+                            returnValue = main.FALSE
+            else:
+                for device in tempFlows:
+                    for flow in device.get( 'flows' ):
+                        if flow.get( 'state' ) != 'ADDED':
+
+                            main.log.info( self.name + ": flow Id: " +
+                                           str( flow.get( 'groupId' ) ) +
+                                           " | state:" +
+                                           str( flow.get( 'state' ) ) )
+                            returnValue = main.FALSE
 
             return returnValue
         except TypeError:
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index b6be81a..0202a15 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -272,57 +272,61 @@
         returnVal = main.TRUE
         ssh_newkey = 'Are you sure you want to continue connecting'
         refused = "ssh: connect to host " + \
-            ipAddress + " port 22: Connection refused"
+                  ipAddress + " port 22: Connection refused"
 
         if direction == "from":
             cmd = 'scp ' + str( userName ) + '@' + str( ipAddress ) + ':' + \
-            str( filePath ) + ' ' + str( dstPath )
+                  str( filePath ) + ' ' + str( dstPath )
         elif direction == "to":
             cmd = 'scp ' + str( filePath ) + ' ' + str( userName ) + \
-            '@' + str( ipAddress ) + ':' + str( dstPath )
+                  '@' + str( ipAddress ) + ':' + str( dstPath )
         else:
             main.log.debug( "Wrong direction using secure copy command!" )
             return main.FALSE
 
         main.log.info( "Sending: " + cmd )
         self.handle.sendline( cmd )
-        i = self.handle.expect( [
-                            ssh_newkey,
-                            'password:',
-                            "100%",
-                            pexpect.EOF,
-                            pexpect.TIMEOUT,
-                            refused ],
-                            120 )
+        i = 0
+        while i < 2:
+            i = self.handle.expect( [
+                                ssh_newkey,
+                                'password:',
+                                "100%",
+                                refused,
+                                "No such file or directory",
+                                pexpect.EOF,
+                                pexpect.TIMEOUT ],
+                                120 )
 
-        if i == 0:
-            main.log.info( "ssh key confirmation received, send yes" )
-            self.handle.sendline( 'yes' )
-            i = self.handle.expect( [ ssh_newkey, 'password:', pexpect.EOF ] )
-        if i == 1:
-            main.log.info( "ssh connection asked for password, gave password" )
-            self.handle.sendline( pwd )
-            # self.handle.expect( userName )
-        if i == 2:
-            main.log.info( "Secure copy successful\n" + self.handle.before  )
-            returnVal = main.TRUE
-        elif i == 3:
-            main.log.error( "Pexpect.EOF found!!!" )
-            main.cleanup()
-            main.exit()
-        elif i == 4:  # timeout
-            main.log.error(
-                "No route to the Host " +
-                userName +
-                "@" +
-                ipAddress )
-            returnVal = main.FALSE
-        elif i == 5:
-            main.log.error(
-                "ssh: connect to host " +
-                ipAddress +
-                " port 22: Connection refused" )
-            returnVal = main.FALSE
+            if i == 0:  # ask for ssh key confirmation
+                main.log.info( "ssh key confirmation received, sending yes" )
+                self.handle.sendline( 'yes' )
+            elif i == 1:  # Asked for ssh password
+                main.log.info( "ssh connection asked for password, gave password" )
+                self.handle.sendline( pwd )
+            elif i == 2:  # File finished transfering
+                main.log.info( "Secure copy successful" )
+                returnVal = main.TRUE
+            elif i == 3:  # Connection refused
+                main.log.error(
+                    "ssh: connect to host " +
+                    ipAddress +
+                    " port 22: Connection refused" )
+                returnVal = main.FALSE
+            elif i == 4:  # File Not found
+                main.log.error( "No such file found" )
+                returnVal = main.FALSE
+            elif i == 5:  # EOF
+                main.log.error( "Pexpect.EOF found!!!" )
+                main.cleanup()
+                main.exit()
+            elif i == 6:  # timeout
+                main.log.error(
+                    "No route to the Host " +
+                    userName +
+                    "@" +
+                    ipAddress )
+                returnVal = main.FALSE
 
         self.handle.sendline( "" )
         self.handle.expect( "$" )
diff --git a/TestON/tests/FUNCvirNetNB/FUNCvirNetNB.params b/TestON/tests/FUNCvirNetNB/FUNCvirNetNB.params
index 7f0e058..8e7206e 100644
--- a/TestON/tests/FUNCvirNetNB/FUNCvirNetNB.params
+++ b/TestON/tests/FUNCvirNetNB/FUNCvirNetNB.params
@@ -11,7 +11,7 @@
     # 9 - Update Virtualport northbound test
     #10 - Delete Virtualport northbound test
 
-    <testcases>1,2,3,4,5,6,7</testcases>
+    <testcases>1,2,3,4,5,6,7,8,9,10</testcases>
 
     <SLEEP>
         <startup>15</startup>
diff --git a/TestON/tests/FUNCvirNetNB/FUNCvirNetNB.py b/TestON/tests/FUNCvirNetNB/FUNCvirNetNB.py
index 4d65bc6..7577d49 100644
--- a/TestON/tests/FUNCvirNetNB/FUNCvirNetNB.py
+++ b/TestON/tests/FUNCvirNetNB/FUNCvirNetNB.py
@@ -57,16 +57,22 @@
         cellName = main.params['ENV']['cellName']
         ipList = os.getenv( main.params['CTRL']['ip1'] )
 
-        main.step("Create cell file")
+        main.step("Create cell file and apply to environment")
         cellAppString = main.params['ENV']['cellApps']
         main.ONOSbench.createCellFile(main.ONOSbench.ip_address,cellName,
                                       main.Mininet1.ip_address,
                                       cellAppString,ipList )
 
-        main.step( "Applying cell variable to environment" )
         cellResult = main.ONOSbench.setCell(cellName)
         verifyResult = main.ONOSbench.verifyCell()
 
+        stepResult = cellResult and verifyResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully applied cell to " + \
+                                        "environment",
+                                 onfail="Failed to apply cell to environment " )
+
         #FIXME:this is short term fix
         main.log.info( "Removing raft logs" )
         main.ONOSbench.onosRemoveRaftLogs()
@@ -97,7 +103,7 @@
 
         cleanInstallResult = main.TRUE
         gitPullResult = main.TRUE
-        main.step( "Git checkout and pull " + gitBranch )
+        main.log.info( "Git checkout and pull " + gitBranch )
         if PULLCODE:
             main.ONOSbench.gitCheckout ( gitBranch )
             gitPullResult = main.ONOSbench.gitPull()
@@ -209,7 +215,7 @@
         port = main.params['HTTP']['port']
         path = main.params['HTTP']['path']
 
-        main.step( "Generate Post Data" )
+        main.log.info( "Generate Post Data" )
         network = NetworkData()
         network.id = '030d6d3d-fa36-45bf-ae2b-4f4bc43a54dc'
         network.tenant_id = '26cd996094344a0598b0a1af1d525cdc'
@@ -283,7 +289,7 @@
         port = main.params['HTTP']['port']
         path = main.params['HTTP']['path']
 
-        main.step( "Generate Post Data" )
+        main.log.info( "Generate Post Data" )
         network = NetworkData()
         network.id = '030d6d3d-fa36-45bf-ae2b-4f4bc43a54dc'
         network.tenant_id = '26cd996094344a0598b0a1af1d525cdc'
@@ -370,7 +376,7 @@
         port = main.params['HTTP']['port']
         path = main.params['HTTP']['path']
 
-        main.step( "Generate Post Data" )
+        main.log.info( "Generate Post Data" )
         network = NetworkData()
         network.id = '030d6d3d-fa36-45bf-ae2b-4f4bc43a54dc'
         network.tenant_id = '26cd996094344a0598b0a1af1d525cdc'
@@ -431,7 +437,7 @@
         port = main.params['HTTP']['port']
         path = main.params['HTTP']['path']
 
-        main.step( "Generate Post Data" )
+        main.log.info( "Generate Post Data" )
         network = NetworkData()
         network.id = '030d6d3d-fa36-45bf-ae2b-4f4bc43a54dc'
         network.tenant_id = '26cd996094344a0598b0a1af1d525cdc'
@@ -520,7 +526,7 @@
         port = main.params['HTTP']['port']
         path = main.params['HTTP']['path']
 
-        main.step( "Generate Post Data" )
+        main.log.info( "Generate Post Data" )
         network = NetworkData()
         network.id = '030d6d3d-fa36-45bf-ae2b-4f4bc43a54dc'
         network.tenant_id = '26cd996094344a0598b0a1af1d525cdc'
@@ -626,7 +632,7 @@
         port = main.params['HTTP']['port']
         path = main.params['HTTP']['path']
 
-        main.step( "Generate Post Data" )
+        main.log.info( "Generate Post Data" )
         network = NetworkData()
         network.id = '030d6d3d-fa36-45bf-ae2b-4f4bc43a54dc'
         network.tenant_id = '26cd996094344a0598b0a1af1d525cdc'
@@ -703,7 +709,7 @@
         httpport = main.params['HTTP']['port']
         path = main.params['HTTP']['path']
 
-        main.step( "Generate Post Data" )
+        main.log.info( "Generate Post Data" )
         network = NetworkData()
         network.id = '030d6d3d-fa36-45bf-ae2b-4f4bc43a54dc'
         network.tenant_id = '26cd996094344a0598b0a1af1d525cdc'
@@ -811,7 +817,7 @@
         httpport = main.params['HTTP']['port']
         path = main.params['HTTP']['path']
 
-        main.step( "Generate Post Data" )
+        main.log.info( "Generate Post Data" )
         network = NetworkData()
         network.id = '030d6d3d-fa36-45bf-ae2b-4f4bc43a54dc'
         network.tenant_id = '26cd996094344a0598b0a1af1d525cdc'
@@ -934,7 +940,7 @@
         httpport = main.params['HTTP']['port']
         path = main.params['HTTP']['path']
 
-        main.step( "Generate Post Data" )
+        main.log.info( "Generate Post Data" )
         network = NetworkData()
         network.id = '030d6d3d-fa36-45bf-ae2b-4f4bc43a54dc'
         network.tenant_id = '26cd996094344a0598b0a1af1d525cdc'
@@ -1007,4 +1013,4 @@
                 expect='200',
                 actual=deletestatus,
                 onpass="Delete Network Success",
-                onfail="Delete Network Failed" )
\ No newline at end of file
+                onfail="Delete Network Failed" )
diff --git a/TestON/tests/FUNCvirNetNB/dependencies/Nbdata.py b/TestON/tests/FUNCvirNetNB/dependencies/Nbdata.py
index df2121e..9b3b978 100644
--- a/TestON/tests/FUNCvirNetNB/dependencies/Nbdata.py
+++ b/TestON/tests/FUNCvirNetNB/dependencies/Nbdata.py
@@ -184,7 +184,7 @@
         if self.allowedAddressPairs != '':
             Dicdata['allowed_address_pairs'] = self.allowedAddressPairs
         if self.deviceOwner != '':
-            Dicdata['device_owner'] = self.deviceOwner            
+            Dicdata['device_owner'] = self.deviceOwner
         if self.securityGroups != '':
             Dicdata['security_groups'] = self.securityGroups
         if self.adminStateUp != '':
diff --git a/TestON/tests/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HAclusterRestart/HAclusterRestart.py
index 1abca54..3026c1e 100644
--- a/TestON/tests/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HAclusterRestart/HAclusterRestart.py
@@ -510,19 +510,22 @@
         # FIXME: Once we have a host discovery mechanism, use that instead
         # REACTIVE FWD test
         pingResult = main.FALSE
-        for i in range(2):  # Retry if pingall fails first time
-            time1 = time.time()
+        passMsg = "Reactive Pingall test passed"
+        time1 = time.time()
+        pingResult = main.Mininet1.pingall()
+        time2 = time.time()
+        if not pingResult:
+            main.log.warn("First pingall failed. Trying again...")
             pingResult = main.Mininet1.pingall()
-            if i == 0:
-                utilities.assert_equals(
-                    expect=main.TRUE,
-                    actual=pingResult,
-                    onpass="Reactive Pingall test passed",
-                    onfail="Reactive Pingall failed, " +
-                           "one or more ping pairs failed" )
-            time2 = time.time()
-            main.log.info( "Time for pingall: %2f seconds" %
-                           ( time2 - time1 ) )
+            passMsg += " on the second try"
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=pingResult,
+            onpass= passMsg,
+            onfail="Reactive Pingall failed, " +
+                   "one or more ping pairs failed" )
+        main.log.info( "Time for pingall: %2f seconds" %
+                       ( time2 - time1 ) )
         # timeout for fwd flows
         time.sleep( 11 )
         # uninstall onos-app-fwd
@@ -2699,15 +2702,14 @@
         main.Mininet2.stopTcpdump()
 
         testname = main.TEST
-        main.log.error( main.params[ 'BACKUP' ][ 'ENABLED' ] )
         if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
             main.step( "Copying MN pcap and ONOS log files to test station" )
             teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
             teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
-            # NOTE: MN Pcap file is being saved to ~/packet_captures
-            #       scp this file as MN and TestON aren't necessarily the same vm
-            # FIXME: scp
-            # mn files
+            # NOTE: MN Pcap file is being saved to logdir.
+            #       We scp this file as MN and TestON aren't necessarily the same vm
+
+            # FIXME: To be replaced with a Jenkin's post script
             # TODO: Load these from params
             # NOTE: must end in /
             logFolder = "/opt/onos/log/"
@@ -2715,14 +2717,9 @@
             # NOTE: must end in /
             for f in logFiles:
                 for node in main.nodes:
-                    # scp sdn@<ip>:<file path> <test's log path>/<node name>-<orig filename>
-                    main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
-                                                    ":" + logFolder + f + " " +
-                                                    main.logdir + "/" +
-                                                    node.name + "-" + f )
-                    main.ONOSbench.handle.expect( "\$ " )
-                    print main.ONOSbench.handle.before
-
+                    dstName =  main.logdir + "/" + node.name + "-" + f
+                    main.ONOSbench.secureCopy( node.user_name, node.ip_address,
+                                               logFolder + f, dstName )
             # std*.log's
             # NOTE: must end in /
             logFolder = "/opt/onos/var/"
@@ -2730,17 +2727,9 @@
             # NOTE: must end in /
             for f in logFiles:
                 for node in main.nodes:
-                    # scp sdn@<ip>:<file path> <test's log path>/<node name>-<orig filename>
-                    main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
-                                                    ":" + logFolder + f + " " +
-                                                    main.logdir + "/" +
-                                                    node.name + "-" + f )
-                    main.ONOSbench.handle.expect( "\$ " )
-                    print main.ONOSbench.handle.before
-            # sleep so scp can finish
-            time.sleep( 10 )
-            # main.step( "Packing and rotating pcap archives" )
-            # os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+                    dstName =  main.logdir + "/" + node.name + "-" + f
+                    main.ONOSbench.secureCopy( node.user_name, node.ip_address,
+                                               logFolder + f, dstName )
         else:
             main.log.debug( "skipping saving log files" )
 
diff --git a/TestON/tests/HAminorityRestart/HAminorityRestart.py b/TestON/tests/HAminorityRestart/HAminorityRestart.py
index 855f380..69c0645 100644
--- a/TestON/tests/HAminorityRestart/HAminorityRestart.py
+++ b/TestON/tests/HAminorityRestart/HAminorityRestart.py
@@ -494,19 +494,22 @@
         # FIXME: Once we have a host discovery mechanism, use that instead
         # REACTIVE FWD test
         pingResult = main.FALSE
-        for i in range(2):  # Retry if pingall fails first time
-            time1 = time.time()
+        passMsg = "Reactive Pingall test passed"
+        time1 = time.time()
+        pingResult = main.Mininet1.pingall()
+        time2 = time.time()
+        if not pingResult:
+            main.log.warn("First pingall failed. Trying again...")
             pingResult = main.Mininet1.pingall()
-            if i == 0:
-                utilities.assert_equals(
-                    expect=main.TRUE,
-                    actual=pingResult,
-                    onpass="Reactive Pingall test passed",
-                    onfail="Reactive Pingall failed, " +
-                           "one or more ping pairs failed" )
-            time2 = time.time()
-            main.log.info( "Time for pingall: %2f seconds" %
-                           ( time2 - time1 ) )
+            passMsg += " on the second try"
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=pingResult,
+            onpass= passMsg,
+            onfail="Reactive Pingall failed, " +
+                   "one or more ping pairs failed" )
+        main.log.info( "Time for pingall: %2f seconds" %
+                       ( time2 - time1 ) )
         # timeout for fwd flows
         time.sleep( 11 )
         # uninstall onos-app-fwd
@@ -1671,6 +1674,12 @@
         assert main.CLIs, "main.CLIs not defined"
         assert main.nodes, "main.nodes not defined"
         main.case( "Restart minority of ONOS nodes" )
+
+        main.step( "Checking ONOS Logs for errors" )
+        for node in main.nodes:
+            main.log.debug( "Checking logs for errors on " + node.name + ":" )
+            main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
+
         main.step( "Killing 3 ONOS nodes" )
         killTime = time.time()
         # TODO: Randomize these nodes or base this on partitions
@@ -2095,10 +2104,6 @@
                                 " and ONOS"
 
         main.step( "Comparing ONOS topology to MN" )
-        devicesResults = main.TRUE
-        linksResults = main.TRUE
-        hostsResults = main.TRUE
-        hostAttachmentResults = True
         topoResult = main.FALSE
         elapsed = 0
         count = 0
@@ -2106,6 +2111,10 @@
         startTime = time.time()
         # Give time for Gossip to work
         while topoResult == main.FALSE and elapsed < 60:
+            devicesResults = main.TRUE
+            linksResults = main.TRUE
+            hostsResults = main.TRUE
+            hostAttachmentResults = True
             count += 1
             cliStart = time.time()
             devices = []
@@ -2638,49 +2647,36 @@
         main.Mininet2.stopTcpdump()
 
         testname = main.TEST
-        if main.params[ 'BACKUP' ] == "True":
+        if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
             main.step( "Copying MN pcap and ONOS log files to test station" )
             teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
             teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
-            # NOTE: MN Pcap file is being saved to ~/packet_captures
-            #       scp this file as MN and TestON aren't necessarily the same vm
-            # FIXME: scp
-            # mn files
+            # NOTE: MN Pcap file is being saved to logdir.
+            #       We scp this file as MN and TestON aren't necessarily the same vm
+
+            # FIXME: To be replaced with a Jenkin's post script
             # TODO: Load these from params
             # NOTE: must end in /
             logFolder = "/opt/onos/log/"
             logFiles = [ "karaf.log", "karaf.log.1" ]
             # NOTE: must end in /
-            dstDir = "~/packet_captures/"
             for f in logFiles:
                 for node in main.nodes:
-                    main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
-                                                    ":" + logFolder + f + " " +
-                                                    teststationUser + "@" +
-                                                    teststationIP + ":" +
-                                                    dstDir + str( testname ) +
-                                                    "-" + node.name + "-" + f )
-                    main.ONOSbench.handle.expect( "\$" )
-
+                    dstName =  main.logdir + "/" + node.name + "-" + f
+                    main.ONOSbench.secureCopy( node.user_name, node.ip_address,
+                                               logFolder + f, dstName )
             # std*.log's
             # NOTE: must end in /
             logFolder = "/opt/onos/var/"
             logFiles = [ "stderr.log", "stdout.log" ]
             # NOTE: must end in /
-            dstDir = "~/packet_captures/"
             for f in logFiles:
                 for node in main.nodes:
-                    main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
-                                                    ":" + logFolder + f + " " +
-                                                    teststationUser + "@" +
-                                                    teststationIP + ":" +
-                                                    dstDir + str( testname ) +
-                                                    "-" + node.name + "-" + f )
-                    main.ONOSbench.handle.expect( "\$" )
-            # sleep so scp can finish
-            time.sleep( 10 )
-            main.step( "Packing and rotating pcap archives" )
-            os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+                    dstName =  main.logdir + "/" + node.name + "-" + f
+                    main.ONOSbench.secureCopy( node.user_name, node.ip_address,
+                                               logFolder + f, dstName )
+        else:
+            main.log.debug( "skipping saving log files" )
 
         main.step( "Stopping Mininet" )
         mnResult = main.Mininet1.stopNet()
@@ -2690,9 +2686,8 @@
 
         main.step( "Checking ONOS Logs for errors" )
         for node in main.nodes:
-            print colors[ 'purple' ] + "Checking logs for errors on " + \
-                node.name + ":" + colors[ 'end' ]
-            print main.ONOSbench.checkLogs( node.ip_address, restart=True )
+            main.log.debug( "Checking logs for errors on " + node.name + ":" )
+            main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
 
         try:
             timerLog = open( main.logdir + "/Timers.csv", 'w')
diff --git a/TestON/tests/HAsanity/HAsanity.py b/TestON/tests/HAsanity/HAsanity.py
index 6ceaead..e136d5e 100644
--- a/TestON/tests/HAsanity/HAsanity.py
+++ b/TestON/tests/HAsanity/HAsanity.py
@@ -494,19 +494,22 @@
         # FIXME: Once we have a host discovery mechanism, use that instead
         # REACTIVE FWD test
         pingResult = main.FALSE
-        for i in range(2):  # Retry if pingall fails first time
-            time1 = time.time()
+        passMsg = "Reactive Pingall test passed"
+        time1 = time.time()
+        pingResult = main.Mininet1.pingall()
+        time2 = time.time()
+        if not pingResult:
+            main.log.warn("First pingall failed. Trying again...")
             pingResult = main.Mininet1.pingall()
-            if i == 0:
-                utilities.assert_equals(
-                    expect=main.TRUE,
-                    actual=pingResult,
-                    onpass="Reactive Pingall test passed",
-                    onfail="Reactive Pingall failed, " +
-                           "one or more ping pairs failed" )
-            time2 = time.time()
-            main.log.info( "Time for pingall: %2f seconds" %
-                           ( time2 - time1 ) )
+            passMsg += " on the second try"
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=pingResult,
+            onpass= passMsg,
+            onfail="Reactive Pingall failed, " +
+                   "one or more ping pairs failed" )
+        main.log.info( "Time for pingall: %2f seconds" %
+                       ( time2 - time1 ) )
         # timeout for fwd flows
         time.sleep( 11 )
         # uninstall onos-app-fwd
@@ -515,11 +518,6 @@
         utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
                                  onpass="Uninstall fwd successful",
                                  onfail="Uninstall fwd failed" )
-        '''
-        main.Mininet1.handle.sendline( "py [ h.cmd( \"arping -c 1 10.1.1.1 \" ) for h in net.hosts ] ")
-        import time
-        time.sleep(60)
-        '''
 
         main.step( "Check app ids" )
         threads = []
@@ -2048,10 +2046,6 @@
                                 " and ONOS"
 
         main.step( "Comparing ONOS topology to MN" )
-        devicesResults = main.TRUE
-        linksResults = main.TRUE
-        hostsResults = main.TRUE
-        hostAttachmentResults = True
         topoResult = main.FALSE
         elapsed = 0
         count = 0
@@ -2059,6 +2053,10 @@
         startTime = time.time()
         # Give time for Gossip to work
         while topoResult == main.FALSE and elapsed < 60:
+            devicesResults = main.TRUE
+            linksResults = main.TRUE
+            hostsResults = main.TRUE
+            hostAttachmentResults = True
             count += 1
             cliStart = time.time()
             devices = []
@@ -2601,49 +2599,36 @@
         main.Mininet2.stopTcpdump()
 
         testname = main.TEST
-        if main.params[ 'BACKUP' ] == "True":
+        if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
             main.step( "Copying MN pcap and ONOS log files to test station" )
             teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
             teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
-            # NOTE: MN Pcap file is being saved to ~/packet_captures
-            #       scp this file as MN and TestON aren't necessarily the same vm
-            # FIXME: scp
-            # mn files
+            # NOTE: MN Pcap file is being saved to logdir.
+            #       We scp this file as MN and TestON aren't necessarily the same vm
+
+            # FIXME: To be replaced with a Jenkin's post script
             # TODO: Load these from params
             # NOTE: must end in /
             logFolder = "/opt/onos/log/"
             logFiles = [ "karaf.log", "karaf.log.1" ]
             # NOTE: must end in /
-            dstDir = "~/packet_captures/"
             for f in logFiles:
                 for node in main.nodes:
-                    main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
-                                                    ":" + logFolder + f + " " +
-                                                    teststationUser + "@" +
-                                                    teststationIP + ":" +
-                                                    dstDir + str( testname ) +
-                                                    "-" + node.name + "-" + f )
-                    main.ONOSbench.handle.expect( "\$" )
-
+                    dstName =  main.logdir + "/" + node.name + "-" + f
+                    main.ONOSbench.secureCopy( node.user_name, node.ip_address,
+                                               logFolder + f, dstName )
             # std*.log's
             # NOTE: must end in /
             logFolder = "/opt/onos/var/"
             logFiles = [ "stderr.log", "stdout.log" ]
             # NOTE: must end in /
-            dstDir = "~/packet_captures/"
             for f in logFiles:
                 for node in main.nodes:
-                    main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
-                                                    ":" + logFolder + f + " " +
-                                                    teststationUser + "@" +
-                                                    teststationIP + ":" +
-                                                    dstDir + str( testname ) +
-                                                    "-" + node.name + "-" + f )
-                    main.ONOSbench.handle.expect( "\$" )
-            # sleep so scp can finish
-            time.sleep( 10 )
-            main.step( "Packing and rotating pcap archives" )
-            os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+                    dstName =  main.logdir + "/" + node.name + "-" + f
+                    main.ONOSbench.secureCopy( node.user_name, node.ip_address,
+                                               logFolder + f, dstName )
+        else:
+            main.log.debug( "skipping saving log files" )
 
         main.step( "Stopping Mininet" )
         mnResult = main.Mininet1.stopNet()
@@ -2653,9 +2638,8 @@
 
         main.step( "Checking ONOS Logs for errors" )
         for node in main.nodes:
-            print colors[ 'purple' ] + "Checking logs for errors on " + \
-                node.name + ":" + colors[ 'end' ]
-            print main.ONOSbench.checkLogs( node.ip_address )
+            main.log.debug( "Checking logs for errors on " + node.name + ":" )
+            main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
 
         try:
             timerLog = open( main.logdir + "/Timers.csv", 'w')
diff --git a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index 8a86647..a2602d1 100644
--- a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -391,19 +391,22 @@
         # FIXME: Once we have a host discovery mechanism, use that instead
         # REACTIVE FWD test
         pingResult = main.FALSE
-        for i in range(2):  # Retry if pingall fails first time
-            time1 = time.time()
+        passMsg = "Reactive Pingall test passed"
+        time1 = time.time()
+        pingResult = main.Mininet1.pingall()
+        time2 = time.time()
+        if not pingResult:
+            main.log.warn("First pingall failed. Trying again...")
             pingResult = main.Mininet1.pingall()
-            if i == 0:
-                utilities.assert_equals(
-                    expect=main.TRUE,
-                    actual=pingResult,
-                    onpass="Reactive Pingall test passed",
-                    onfail="Reactive Pingall failed, " +
-                           "one or more ping pairs failed" )
-            time2 = time.time()
-            main.log.info( "Time for pingall: %2f seconds" %
-                           ( time2 - time1 ) )
+            passMsg += " on the second try"
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=pingResult,
+            onpass= passMsg,
+            onfail="Reactive Pingall failed, " +
+                   "one or more ping pairs failed" )
+        main.log.info( "Time for pingall: %2f seconds" %
+                       ( time2 - time1 ) )
         # timeout for fwd flows
         time.sleep( 11 )
         # uninstall onos-app-fwd
@@ -1184,6 +1187,12 @@
         main.case( "Restart ONOS node" )
         main.caseExplanation = "Killing ONOS process and restart cli " +\
                                 "sessions once onos is up."
+
+        main.step( "Checking ONOS Logs for errors" )
+        for node in main.nodes:
+            main.log.debug( "Checking logs for errors on " + node.name + ":" )
+            main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
+
         main.step( "Killing ONOS processes" )
         killResult = main.ONOSbench.onosKill( main.nodes[0].ip_address )
         start = time.time()
@@ -1423,10 +1432,6 @@
                                 " and ONOS"
 
         main.step( "Comparing ONOS topology to MN" )
-        devicesResults = main.TRUE
-        linksResults = main.TRUE
-        hostsResults = main.TRUE
-        hostAttachmentResults = True
         topoResult = main.FALSE
         elapsed = 0
         count = 0
@@ -1434,6 +1439,10 @@
         startTime = time.time()
         # Give time for Gossip to work
         while topoResult == main.FALSE and elapsed < 60:
+            devicesResults = main.TRUE
+            linksResults = main.TRUE
+            hostsResults = main.TRUE
+            hostAttachmentResults = True
             count += 1
             cliStart = time.time()
             devices = []
@@ -1765,45 +1774,36 @@
         main.Mininet2.stopTcpdump()
 
         testname = main.TEST
-        if main.params[ 'BACKUP' ] == "True":
+        if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
             main.step( "Copying MN pcap and ONOS log files to test station" )
             teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
             teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
-            # NOTE: MN Pcap file is being saved to ~/packet_captures
-            #       scp this file as MN and TestON aren't necessarily the same vm
-            # FIXME: scp
-            # mn files
+            # NOTE: MN Pcap file is being saved to logdir.
+            #       We scp this file as MN and TestON aren't necessarily the same vm
+
+            # FIXME: To be replaced with a Jenkin's post script
             # TODO: Load these from params
             # NOTE: must end in /
             logFolder = "/opt/onos/log/"
             logFiles = [ "karaf.log", "karaf.log.1" ]
             # NOTE: must end in /
-            dstDir = "~/packet_captures/"
             for f in logFiles:
-                main.ONOSbench.handle.sendline( "scp sdn@" + main.nodes[0].ip_address + ":" +
-                                                logFolder + f + " " +
-                                                teststationUser + "@" +
-                                                teststationIP + ":" + dstDir +
-                                                str( testname ) + "-ONOS1-" + f )
-                main.ONOSbench.handle.expect( "\$" )
-
+                for node in main.nodes:
+                    dstName =  main.logdir + "/" + node.name + "-" + f
+                    main.ONOSbench.secureCopy( node.user_name, node.ip_address,
+                                               logFolder + f, dstName )
             # std*.log's
             # NOTE: must end in /
             logFolder = "/opt/onos/var/"
             logFiles = [ "stderr.log", "stdout.log" ]
             # NOTE: must end in /
-            dstDir = "~/packet_captures/"
             for f in logFiles:
-                main.ONOSbench.handle.sendline( "scp sdn@" + main.nodes[0].ip_address + ":" +
-                                                logFolder + f + " " +
-                                                teststationUser + "@" +
-                                                teststationIP + ":" + dstDir +
-                                                str( testname ) + "-ONOS1-" + f )
-                main.ONOSbench.handle.expect( "\$" )
-            # sleep so scp can finish
-            time.sleep( 10 )
-            main.step( "Packing and rotating pcap archives" )
-            os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+                for node in main.nodes:
+                    dstName =  main.logdir + "/" + node.name + "-" + f
+                    main.ONOSbench.secureCopy( node.user_name, node.ip_address,
+                                               logFolder + f, dstName )
+        else:
+            main.log.debug( "skipping saving log files" )
 
         main.step( "Stopping Mininet" )
         mnResult = main.Mininet1.stopNet()
@@ -1812,9 +1812,9 @@
                                  onfail="MN cleanup NOT successful" )
 
         main.step( "Checking ONOS Logs for errors" )
-        print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
-            colors[ 'end' ]
-        print main.ONOSbench.checkLogs( main.nodes[0].ip_address, restart=True )
+        for node in main.nodes:
+            main.log.debug( "Checking logs for errors on " + node.name + ":" )
+            main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
 
         try:
             timerLog = open( main.logdir + "/Timers.csv", 'w')
diff --git a/TestON/tests/USECASE_SdnipI2/Dependency/Functions.py b/TestON/tests/USECASE_SdnipI2/Dependency/Functions.py
index 732ff41..1b93f49 100644
--- a/TestON/tests/USECASE_SdnipI2/Dependency/Functions.py
+++ b/TestON/tests/USECASE_SdnipI2/Dependency/Functions.py
@@ -39,3 +39,105 @@
         expect = intentNumExpected, actual = intentNumActual,
         onpass = "***P2P intent number is correct!***",
         onfail = "***P2P intent number is wrong!***" )
+
+def checkFlowNum( main, switch, flowNumExpected ):
+    main.step( "Check flow entry number in " + switch )
+    main.log.info( "Flow number expected:" )
+    main.log.info( flowNumExpected )
+    main.log.info( "Flow number actual:" )
+    flowNumActual = main.Mininet.getSwitchFlowCount( switch )
+    main.log.info( flowNumActual )
+    utilities.assertEquals( \
+        expect = flowNumExpected, actual = flowNumActual,
+        onpass = "***Flow number in " + switch + " is correct!***",
+        onfail = "***Flow number in " + switch + " is wrong!***" )
+
+
+def pingSpeakerToPeer( main, speakers = ["speaker1"],
+                       peers = ["peer64514", "peer64515", "peer64516"],
+                       expectAllSuccess = True ):
+    """
+    Carry out ping test between each BGP speaker and peer pair
+    Optional argument:
+        * speakers - BGP speakers
+        * peers - BGP peers
+        * expectAllSuccess - boolean indicating if you expect all results
+        succeed if True, otherwise expect all results fail if False
+    """
+    if len( speakers ) == 0:
+        main.log.error( "Parameter speakers can not be empty." )
+        main.clearUp()
+        main.exit()
+    if len( peers ) == 0:
+        main.log.error( "Parameter speakers can not be empty." )
+        main.clearUp()
+        main.exit()
+
+    if expectAllSuccess:
+        main.step( "Check ping between BGP peers and speakers, expect all tests\
+        will SUCCEED" )
+    else:
+        main.step( "Check ping between BGP peers and speakers, expect all tests\
+        will FAIL" )
+
+    result = True
+    if expectAllSuccess:
+        for speaker in speakers:
+            for peer in peers:
+                tmpResult = main.Mininet.pingHost( src = speaker,
+                                                   target = peer )
+                result = result and ( tmpResult == main.TRUE )
+    else:
+        for speaker in speakers:
+            for peer in peers:
+                tmpResult = main.Mininet.pingHost( src = speaker,
+                                                   target = peer )
+
+    utilities.assert_equals( expect = True, actual = result,
+                             onpass = "Ping test results are expected",
+                             onfail = "Ping test results are Not expected" )
+
+    if result == False:
+        main.clearUp()
+        main.exit()
+
+
+def pingHostToHost( main, hosts = ["host64514", "host64515", "host64516"],
+                expectAllSuccess = True ):
+    """
+    Carry out ping test between each BGP host pair
+    Optional argument:
+        * hosts - hosts behind BGP peer routers
+        * expectAllSuccess - boolean indicating if you expect all results
+        succeed if True, otherwise expect all results fail if False
+    """
+    main.step( "Check ping between each host pair" )
+    if len( hosts ) == 0:
+        main.log.error( "Parameter hosts can not be empty." )
+        main.clearUp()
+        main.exit()
+
+    result = True
+    if expectAllSuccess:
+        for srcHost in hosts:
+            for targetHost in hosts:
+                if srcHost != targetHost:
+                    tmpResult = main.Mininet.pingHost( src = srcHost,
+                                                       target = targetHost )
+                    result = result and ( tmpResult == main.TRUE )
+    else:
+        for srcHost in hosts:
+            for targetHost in hosts:
+                if srcHost != targetHost:
+                    tmpResult = main.Mininet.pingHost( src = srcHost,
+                                                       target = targetHost )
+                    result = result and ( tmpResult == main.FALSE )
+
+    utilities.assert_equals( expect = True, actual = result,
+                             onpass = "Ping test results are expected",
+                             onfail = "Ping test results are Not expected" )
+
+    if result == False:
+        main.cleanup()
+        main.exit()
+
diff --git a/TestON/tests/USECASE_SdnipI2/USECASE_SdnipI2.params b/TestON/tests/USECASE_SdnipI2/USECASE_SdnipI2.params
index fd65488..0d42860 100644
--- a/TestON/tests/USECASE_SdnipI2/USECASE_SdnipI2.params
+++ b/TestON/tests/USECASE_SdnipI2/USECASE_SdnipI2.params
@@ -1,6 +1,6 @@
 <PARAMS>
 
-    <testcases>100, 101, 102, 7, 8, 1, 4</testcases>
+    <testcases>100, 101, 102, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10</testcases>
 
     #Environment variables
     <ENV>
diff --git a/TestON/tests/USECASE_SdnipI2/USECASE_SdnipI2.py b/TestON/tests/USECASE_SdnipI2/USECASE_SdnipI2.py
index c1ff688..ebc4545 100644
--- a/TestON/tests/USECASE_SdnipI2/USECASE_SdnipI2.py
+++ b/TestON/tests/USECASE_SdnipI2/USECASE_SdnipI2.py
@@ -143,24 +143,10 @@
         ping test from 3 bgp peers to BGP speaker
         '''
 
-        m2SIntentsNumberActual = main.ONOScli.m2SIntentInstalledNumber()
-        main.log.info( "MultiPointToSinglePoint intent number actual is:" )
-        main.log.info( m2SIntentsNumberActual )
-
         main.case( "This case is to check ping between BGP peers and speakers" )
-        result1 = main.Mininet.pingHost( src = "speaker1", target = "peer64514" )
-        result2 = main.Mininet.pingHost( src = "speaker1", target = "peer64515" )
-        result3 = main.Mininet.pingHost( src = "speaker1", target = "peer64516" )
-
-
-        caseResult = result1 and result2 and result3
-        utilities.assert_equals( expect = main.TRUE, actual = caseResult,
-                                 onpass = "Speaker1 ping peers successful",
-                                 onfail = "Speaker1 ping peers NOT successful" )
-
-        if caseResult == main.FALSE:
-            main.cleanup()
-            main.exit()
+        main.Functions.pingSpeakerToPeer( main, speakers = ["speaker1"],
+                       peers = ["peer64514", "peer64515", "peer64516"],
+                       expectAllSuccess = True )
 
 
     def CASE2( self, main ):
@@ -232,24 +218,23 @@
             onfail = "***MultiPointToSinglePoint Intent Num in SDN-IP is \
             wrong!***" )
 
+        main.step( "Check whether all flow status are ADDED" )
+        utilities.assertEquals( \
+            expect = main.TRUE,
+            actual = main.ONOScli.checkFlowsState( isPENDING_ADD = False ),
+            onpass = "***Flow status is correct!***",
+            onfail = "***Flow status is wrong!***" )
+
 
     def CASE4( self, main ):
         '''
         Ping test in data plane for each route
         '''
-        main.case( "This case is to check ping for each route" )
-        result1 = main.Mininet.pingHost( src = "host64514", target = "host64515" )
-        result2 = main.Mininet.pingHost( src = "host64515", target = "host64516" )
-        result3 = main.Mininet.pingHost( src = "host64514", target = "host64516" )
-
-        caseResult = result1 and result2 and result3
-        utilities.assert_equals( expect = main.TRUE, actual = caseResult,
-                                 onpass = "Ping test for each route successful",
-                                 onfail = "Ping test for each route NOT successful" )
-
-        if caseResult == main.FALSE:
-            main.cleanup()
-            main.exit()
+        main.case( "This case is to check ping for each route, \
+        all hosts behind BGP peers" )
+        main.Functions.pingHostToHost( main,
+                        hosts = ["host64514", "host64515", "host64516"],
+                        expectAllSuccess = True )
 
 
     def CASE5( self, main ):
@@ -291,8 +276,23 @@
             main.log.info( "Bring down link failed!!!" )
             main.exit();
 
+        main.step( "Check whether all flow status are ADDED" )
+        utilities.assertEquals( \
+            expect = main.TRUE,
+            actual = main.ONOScli.checkFlowsState( isPENDING_ADD = False ),
+            onpass = "***Flow status is correct!***",
+            onfail = "***Flow status is wrong!***" )
 
-    def CASE6(self, main):
+        # Ping test
+        main.Functions.pingSpeakerToPeer( main, speakers = ["speaker1"],
+                       peers = ["peer64514", "peer64515", "peer64516"],
+                       expectAllSuccess = False )
+        main.Functions.pingHostToHost( main,
+                        hosts = ["host64514", "host64515", "host64516"],
+                        expectAllSuccess = False )
+
+
+    def CASE6( self, main ):
         '''
         Recover links to peers one by one, check routes/intents
         '''
@@ -330,19 +330,30 @@
         else:
             main.log.info( "Bring up link failed!!!" )
             main.exit();
-        '''
-        Note: at the end of this test case, we should carry out ping test.
-        So we run CASE4 again after CASE6
-        '''
+
+        main.step( "Check whether all flow status are ADDED" )
+        utilities.assertEquals( \
+            expect = main.TRUE,
+            actual = main.ONOScli.checkFlowsState( isPENDING_ADD = False ),
+            onpass = "***Flow status is correct!***",
+            onfail = "***Flow status is wrong!***" )
+
+        # Ping test
+        main.Functions.pingSpeakerToPeer( main, speakers = ["speaker1"],
+                       peers = ["peer64514", "peer64515", "peer64516"],
+                       expectAllSuccess = True )
+        main.Functions.pingHostToHost( main,
+                        hosts = ["host64514", "host64515", "host64516"],
+                        expectAllSuccess = True )
 
 
-    def CASE7(self, main):
+    def CASE7( self, main ):
         '''
-        shut down a edge switch, check P-2-P and M-2-S intents, ping test
+        Shut down a edge switch, check P-2-P and M-2-S intents, ping test
         '''
         import time
         main.case( "This case is to stop 1 edge switch,\
-        check P-2-P and M-2-S intents, ping test")
+        check P-2-P and M-2-S intents, ping test" )
         main.step( "Stop sw32" )
         result = main.Mininet.switch( SW = "sw32", OPTION = "stop" )
         if result == main.TRUE:
@@ -354,6 +365,43 @@
             main.log.info( "Stop switch failed!!!" )
             main.exit();
 
+        main.step( "Check ping between hosts behind BGP peers" )
+        result1 = main.Mininet.pingHost( src = "host64514", target = "host64515" )
+        result2 = main.Mininet.pingHost( src = "host64515", target = "host64516" )
+        result3 = main.Mininet.pingHost( src = "host64514", target = "host64516" )
+
+        pingResult1 = ( result1 == main.FALSE ) and ( result2 == main.TRUE ) \
+                                                and ( result3 == main.FALSE )
+        utilities.assert_equals( expect = True, actual = pingResult1,
+                                 onpass = "Ping test result is correct",
+                                 onfail = "Ping test result is wrong" )
+
+        if pingResult1 == False:
+            main.cleanup()
+            main.exit()
+
+        main.step( "Check ping between BGP peers and speakers" )
+        result4 = main.Mininet.pingHost( src = "speaker1", target = "peer64514" )
+        result5 = main.Mininet.pingHost( src = "speaker1", target = "peer64515" )
+        result6 = main.Mininet.pingHost( src = "speaker1", target = "peer64516" )
+
+        pingResult2 = ( result4 == main.FALSE ) and ( result5 == main.TRUE ) \
+                                                and ( result6 == main.TRUE )
+        utilities.assert_equals( expect = True, actual = pingResult2,
+                                 onpass = "Speaker1 ping peers successful",
+                                 onfail = "Speaker1 ping peers NOT successful" )
+
+        if pingResult2 == False:
+            main.cleanup()
+            main.exit()
+
+        main.step( "Check whether all flow status are ADDED" )
+        utilities.assertEquals( \
+            expect = main.TRUE,
+            actual = main.ONOScli.checkFlowsState( isPENDING_ADD = False ),
+            onpass = "***Flow status is correct!***",
+            onfail = "***Flow status is wrong!***" )
+
         '''
         main.step( "Stop sw8" )
         result = main.Mininet.switch( SW = "sw8", OPTION = "stop" )
@@ -379,10 +427,6 @@
             main.log.info( "Stop switch failed!!!" )
             main.exit();
         '''
-        '''
-        ping test between BGP speaker and BGP peers, ping test between hosts
-        behind BGP peers ===
-        '''
 
 
     def CASE8( self, main ):
@@ -407,42 +451,102 @@
             main.cleanup()
             main.exit();
 
+        main.step( "Check whether all flow status are ADDED" )
+        utilities.assertEquals( \
+            expect = main.TRUE,
+            actual = main.ONOScli.checkFlowsState( isPENDING_ADD = False ),
+            onpass = "***Flow status is correct!***",
+            onfail = "***Flow status is wrong!***" )
 
-    def CASE20( self, main ):
+        # Ping test
+        main.Functions.pingSpeakerToPeer( main, speakers = ["speaker1"],
+                       peers = ["peer64514", "peer64515", "peer64516"],
+                       expectAllSuccess = True )
+        main.Functions.pingHostToHost( main,
+                        hosts = ["host64514", "host64515", "host64516"],
+                        expectAllSuccess = True )
+
+
+    def CASE9( self, main ):
         '''
-        ping test from 3 bgp peers to BGP speaker
+        Bring down a switch in best path, check:
+        route number, P2P intent number, M2S intent number, ping test
         '''
-        main.case( "This case is to check ping between BGP peers and speakers" )
-        result1 = main.Mininet.pingHost( src = "speaker1", target = "peer64514" )
-        result2 = main.Mininet.pingHost( src = "speaker1", target = "peer64515" )
-        result3 = main.Mininet.pingHost( src = "speaker1", target = "peer64516" )
+        main.case( "This case is to stop switch in best path, \
+        check route number, P2P intent number, M2S intent number, ping test" )
 
+        main.step( "Check the flow status before stopping sw11" )
+        main.Functions.checkFlowNum( main, "sw11", 13 )
+        main.Functions.checkFlowNum( main, "sw1", 3 )
+        main.Functions.checkFlowNum( main, "sw7", 3 )
+        main.log.info( main.Mininet.checkFlows( "sw11" ) )
+        main.log.info( main.Mininet.checkFlows( "sw1" ) )
+        main.log.info( main.Mininet.checkFlows( "sw7" ) )
 
-        caseResult = result1 or result2 or result3
-        utilities.assert_equals( expect = main.FALSE, actual = caseResult,
-                                 onpass = "Speaker1 failed to ping all peers - Correct",
-                                 onfail = "Speaker1 did not fail to ping all peers- NOT Correct" )
-
-        if caseResult == main.TRUE:
+        main.step( "Stop sw11" )
+        result = main.Mininet.switch( SW = "sw11", OPTION = "stop" )
+        if result:
+            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
+            main.Functions.checkRouteNum( main, 3 )
+            main.Functions.checkM2SintentNum( main, 3 )
+            main.Functions.checkP2PintentNum( main, 18 )
+        else:
+            main.log.info( "Stop switch failed!!!" )
             main.cleanup()
-            main.exit()
+            main.exit();
+
+        main.step( "Check whether all flow status are ADDED" )
+        utilities.assertEquals( \
+            expect = main.TRUE,
+            actual = main.ONOScli.checkFlowsState( isPENDING_ADD = False ),
+            onpass = "***Flow status is correct!***",
+            onfail = "***Flow status is wrong!***" )
+        # Ping test
+        main.Functions.pingSpeakerToPeer( main, speakers = ["speaker1"],
+                       peers = ["peer64514", "peer64515", "peer64516"],
+                       expectAllSuccess = True )
+        main.Functions.pingHostToHost( main,
+                        hosts = ["host64514", "host64515", "host64516"],
+                        expectAllSuccess = True )
 
 
-    def CASE21( self, main ):
+    def CASE10( self, main ):
         '''
-        Ping test in data plane for each route
+        Bring up the switch which was stopped in CASE9, check:
+        route number, P2P intent number, M2S intent number, ping test
         '''
-        main.case( "This case is to check ping for each route" )
-        result1 = main.Mininet.pingHost( src = "host64514", target = "host64515" )
-        result2 = main.Mininet.pingHost( src = "host64515", target = "host64516" )
-        result3 = main.Mininet.pingHost( src = "host64514", target = "host64516" )
+        main.case( "This case is to start switch which was stopped in CASE9, \
+        check route number, P2P intent number, M2S intent number, ping test" )
+        main.step( "Start sw11" )
+        result = main.Mininet.switch( SW = "sw11", OPTION = "start" )
+        if result:
+            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
+            main.Functions.checkRouteNum( main, 3 )
+            main.Functions.checkM2SintentNum( main, 3 )
+            main.Functions.checkP2PintentNum( main, 18 )
 
-        caseResult = result1 or result2 or result3
-        utilities.assert_equals( expect = main.FALSE, actual = caseResult,
-                                 onpass = "Ping test for all routes failed- Correct",
-                                 onfail = "Ping test for all routes NOT failed- NOT Correct" )
-
-        if caseResult == main.TRUE:
+            main.step( "Check the flow status after stop and start sw11" )
+            main.Functions.checkFlowNum( main, "sw11", 3 )
+            main.Functions.checkFlowNum( main, "sw1", 11 )
+            main.Functions.checkFlowNum( main, "sw7", 5 )
+            main.log.info( main.Mininet.checkFlows( "sw11" ) )
+            main.log.info( main.Mininet.checkFlows( "sw1" ) )
+            main.log.info( main.Mininet.checkFlows( "sw7" ) )
+        else:
+            main.log.info( "Start switch failed!!!" )
             main.cleanup()
-            main.exit()
+            main.exit();
 
+        main.step( "Check whether all flow status are ADDED" )
+        utilities.assertEquals( \
+            expect = main.TRUE,
+            actual = main.ONOScli.checkFlowsState( isPENDING_ADD = False ),
+            onpass = "***Flow status is correct!***",
+            onfail = "***Flow status is wrong!***" )
+        # Ping test
+        main.Functions.pingSpeakerToPeer( main, speakers = ["speaker1"],
+                       peers = ["peer64514", "peer64515", "peer64516"],
+                       expectAllSuccess = True )
+        main.Functions.pingHostToHost( main,
+                        hosts = ["host64514", "host64515", "host64516"],
+                        expectAllSuccess = True )