Fix T3 for SRRouting
- Fix the issue that t3-simple doesn't work for external hosts
- Run t3 with both locations for dual-homed hosts
- Add timeout when dumping onos command to file

Change-Id: I2f93fad5c7d95d08f4b8703ecd606a7f070b4fa4
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index c391c22..ed96a23 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -120,13 +120,15 @@
         lib.verifyPing( main,
                         [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
                         [ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ],
-                        stepMsg="Verify reachability from connected internal IPv4 hosts to external IPv4 hosts" )
+                        stepMsg="Verify reachability from connected internal IPv4 hosts to external IPv4 hosts",
+                        t3Simple=False )
     if ipv6:
         lib.verifyPing( main,
                         [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
                         [ h for h in main.externalIpv6Hosts if h not in main.disconnectedExternalIpv6Hosts ],
                         ipv6=True, acceptableFailed=7,
-                        stepMsg="Verify reachability from connected internal IPv6 hosts to external IPv6 hosts" )
+                        stepMsg="Verify reachability from connected internal IPv6 hosts to external IPv6 hosts",
+                        t3Simple=False )
     # Verify disconnected hosts
     if disconnected:
         # Disconnected internal to connected external
@@ -134,25 +136,29 @@
             lib.verifyPing( main, main.disconnectedIpv4Hosts,
                             [ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ],
                             expect=False,
-                            stepMsg="Verify unreachability of disconnected internal IPv4 hosts to connected external IPv4 hosts" )
+                            stepMsg="Verify unreachability of disconnected internal IPv4 hosts to connected external IPv4 hosts",
+                            t3Simple=False )
         if main.disconnectedIpv6Hosts:
             lib.verifyPing( main, main.disconnectedIpv6Hosts,
                             [ h for h in main.externalIpv6Hosts if h not in main.disconnectedExternalIpv6Hosts ],
                             ipv6=True, expect=False,
-                            stepMsg="Verify unreachability of disconnected internal IPv6 hosts to connected external IPv6 hosts" )
+                            stepMsg="Verify unreachability of disconnected internal IPv6 hosts to connected external IPv6 hosts",
+                            t3Simple=False )
         # Connected internal to disconnected external
         if main.disconnectedExternalIpv4Hosts:
             lib.verifyPing( main,
                             [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
                             main.disconnectedExternalIpv4Hosts,
                             expect=False,
-                            stepMsg="Verify unreachability of connected internal IPv4 hosts to disconnected external IPv4 hosts" )
+                            stepMsg="Verify unreachability of connected internal IPv4 hosts to disconnected external IPv4 hosts",
+                            t3Simple=False )
         if main.disconnectedExternalIpv6Hosts:
             lib.verifyPing( main,
                             [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
                             main.disconnectedExternalIpv6Hosts,
                             ipv6=True, expect=False,
-                            stepMsg="Verify unreachability of connected internal IPv6 hosts to disconnected external IPv6 hosts" )
+                            stepMsg="Verify unreachability of connected internal IPv6 hosts to disconnected external IPv6 hosts",
+                            t3Simple=False )
 
 def verifyPing( main, ipv4=True, ipv6=True, disconnected=False, internal=True, external=True ):
     """
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index e9bb8fe..ad39667 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -1179,7 +1179,8 @@
 
     @staticmethod
     def verifyPing( main, srcList, dstList, ipv6=False, expect=True, wait=1,
-                    acceptableFailed=0, skipOnFail=True, stepMsg="Verify Ping" ):
+                    acceptableFailed=0, skipOnFail=True, stepMsg="Verify Ping",
+                    t3Simple=True ):
         """
         Verify reachability from each host in srcList to each host in dstList
         """
@@ -1189,7 +1190,7 @@
         except ( NameError, AttributeError ):
             main.topo = Topology()
         main.step( stepMsg )
-        pingResult = main.topo.ping( srcList, dstList, ipv6, expect, wait, acceptableFailed, skipOnFail, True )
+        pingResult = main.topo.ping( srcList, dstList, ipv6, expect, wait, acceptableFailed, skipOnFail, t3Simple )
         utilities.assert_equals( expect=main.TRUE,
                                  actual=pingResult,
                                  onpass="{}: Pass".format( stepMsg ),
diff --git a/TestON/tests/dependencies/topology.py b/TestON/tests/dependencies/topology.py
index be3b69a..e9fe8d7 100644
--- a/TestON/tests/dependencies/topology.py
+++ b/TestON/tests/dependencies/topology.py
@@ -293,11 +293,16 @@
                 srcIp = unexpectedPing[ 0 ]
                 dstIp = unexpectedPing[ 1 ]
                 main.log.debug( "Collecting t3 with source {} and destination {}".format( srcIp, dstIp ) )
-                cmd = main.Cluster.active( 0 ).CLI.composeT3Command( srcIp, dstIp, ipv6, True, t3Simple )
-                main.log.debug( "t3 command: {}".format( cmd ) )
-                if cmd:
+                cmdList = main.Cluster.active( 0 ).CLI.composeT3Command( srcIp, dstIp, ipv6, True, t3Simple )
+                if not cmdList:
+                    main.log.warn( "Failed to compose T3 command with source {} and destination {}".format( srcIp, dstIp ) )
+                    continue
+                for i in range( 0, len( cmdList ) ):
+                    cmd = cmdList[ i ]
+                    main.log.debug( "t3 command: {}".format( cmd ) )
                     main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, cmd, main.logdir,
-                                                "t3-CASE{}-{}-{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp ) )
+                                                "t3-CASE{}-{}-{}-route{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp, i ),
+                                                timeout=10 )
         return main.FALSE if unexpectedPings else main.TRUE
 
     def sendScapyPackets( self, sender, receiver, pktFilter, pkt, sIface=None, dIface=None, expect=True, acceptableFailed=0, collectT3=True, t3Command="" ):