[ONOS-7635][ONOS-7636] Support for complex scenarios in SRRouting test
- New ping function to execute pings in parallel
- Collect t3 output on ping failures
- Allow expected ping failures

Change-Id: I4492a89bc4c0a581ff2e35bcc1896ddd5ea64a18
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 08d8505..bd00623 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -593,10 +593,9 @@
                 pingResponse += str( str( host ) + " -> " )
                 for temp in dstList:
                     failedPings = 0
-                    if ipv6:
-                        pingCmd = str( host ) + cmd + str( self.getIPAddress( temp, proto='IPv6' ) )
-                    else:
-                        pingCmd = str( host ) + cmd + str( temp )
+                    dstIP = self.getIPAddress( temp, proto='IPV6' if ipv6 else 'IPV4' )
+                    assert dstIP, "Not able to get IP address of host {}".format( temp )
+                    pingCmd = str( host ) + cmd + str( dstIP )
                     while failedPings <= acceptableFailed:
                         main.log.debug( "Pinging from " + str( host ) + " to " + str( temp ) )
                         self.handle.sendline( pingCmd )
@@ -616,7 +615,9 @@
                 pingResponse += "\n"
             main.log.info( pingResponse + "Failed pings: " + str( failedPingsTotal ) )
             return isReachable
-
+        except AssertionError:
+            main.log.exception( "" )
+            return main.FALSE
         except pexpect.TIMEOUT:
             main.log.exception( self.name + ": TIMEOUT exception" )
             response = self.handle.before
@@ -750,6 +751,10 @@
                                           '\*\*\* Unknown command: ' + pingCmd,
                                           pexpect.TIMEOUT ],
                                         timeout=wait + 1 )
+                # For some reason we need to send something
+                # Otherwise ping results won't be read by handle
+                self.handle.sendline( "" )
+                self.handle.expect( self.hostPrompt )
                 if i == 0:
                     response = self.handle.before
                     if not re.search( ',\s0\%\spacket\sloss', response ):
@@ -1268,7 +1273,7 @@
         else:
             main.log.error( "Connection failed to the host" )
 
-    def getIPAddress( self, host , proto='IPV4' ):
+    def getIPAddress( self, host, proto='IPV4' ):
         """
            Verifies the host's ip configured or not."""
         if self.handle:
@@ -1288,7 +1293,7 @@
 
             pattern = ''
             if proto == 'IPV4':
-                pattern = "inet\saddr:(\d+\.\d+\.\d+\.\d+)"
+                pattern = "inet\saddr:(\d+\.\d+\.\d+\.\d+)\s\sBcast"
             else:
                 pattern = "inet6\saddr:\s([\w,:]*)/\d+\sScope:Global"
             ipAddressSearch = re.search( pattern, response )
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 6342efc..76cc415 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -6317,3 +6317,107 @@
         except Exception:
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
+
+    def netcfg( self, jsonFormat=True, args="" ):
+        """
+        Run netcfg cli command with given args
+        """
+        try:
+            cmdStr = "netcfg"
+            if jsonFormat:
+                cmdStr = cmdStr + " -j"
+            if args:
+                cmdStr = cmdStr + " " + str( args )
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            assert "Error executing command" not in handle, handle
+            return handle
+        except AssertionError:
+            main.log.exception( "" )
+            return None
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
+    def composeT3Command( self, sAddr, dAddr, ipv6=False, verbose=True ):
+        """
+        Compose and return t3-troubleshoot cli command for given source and destination addresses
+        Options:
+            sAddr: IP address of the source host
+            dAddr: IP address of the destination host
+        """
+        try:
+            # Collect information of both hosts from onos
+            hosts = self.hosts()
+            hosts = json.loads( hosts )
+            sHost = None
+            dHost = None
+            for host in hosts:
+                if sAddr in host[ "ipAddresses" ]:
+                    sHost = host
+                elif dAddr in host[ "ipAddresses" ]:
+                    dHost = host
+                if sHost and dHost:
+                    break
+            assert sHost, "Not able to find host with IP {}".format( sAddr )
+            assert dHost, "Not able to find host with IP {}".format( dAddr )
+            cmdStr = "t3-troubleshoot"
+            if verbose:
+                cmdStr += " -vv"
+            cmdStr += " -s " + str( sAddr )
+            # TODO: collect t3 for all locations of source host?
+            cmdStr += " -sp " + str( sHost[ "locations" ][ 0 ][ "elementId" ] ) + "/" + str( sHost[ "locations" ][ 0 ][ "port" ] )
+            cmdStr += " -sm " + str( sHost[ "mac" ] )
+            cmdStr += " -d " + str( dAddr )
+            netcfg = self.netcfg( args="devices {}".format( sHost[ "locations" ][ 0 ][ "elementId" ] ) )
+            netcfg = json.loads( netcfg )
+            assert netcfg, "Failed to get netcfg"
+            cmdStr += " -dm " + str( netcfg[ "segmentrouting" ][ "routerMac" ] )
+            if ipv6:
+                cmdStr += " -et ipv6"
+            return cmdStr
+        except AssertionError:
+            main.log.exception( "" )
+            return None
+        except ( KeyError, TypeError ):
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
+    def t3( self, sAddr, dAddr, ipv6=False ):
+        """
+        Run t3-troubleshoot cli command for given source and destination addresses
+        Options:
+            sAddr: IP address of the source host
+            dAddr: IP address of the destination host
+        """
+        try:
+            cmdStr = self.composeT3Command( sAddr, dAddr, ipv6 )
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            assert "Error executing command" not in handle, handle
+            assert "Tracing packet" in handle
+            return handle
+        except AssertionError:
+            main.log.exception( "" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()