[ONOS-7736] Add and verify static routes in host movement cases

Change-Id: I53d818b1e1287cc74d05d89b0c95b62015f38ab1
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index 150d7fd..281e7a5 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -1298,39 +1298,38 @@
         main.Cluster.active( 0 ).CLI.balanceMasters()
         time.sleep( float( main.params[ 'timers' ][ 'balanceMasterSleep' ] ) )
         verify( main )
-
+        # Move an untagged IPv4 host on DAAS-1
         h1v4cfg = '{"of:0000000000000001/7" : { "interfaces" : [ { "ips" : [ "10.1.0.254/24" ], "vlan-untagged": 10 } ] } }'
         lib.moveHost( main, "h1v4", "leaf1", "leaf1", "10.1.0.254", prefixLen=24, cfg=h1v4cfg )
         hostLocations = { "h1v4": "of:0000000000000001/7" }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
-
+        # Move an untagged IPv6 host on DAAS-1
         h1v6cfg = '{"of:0000000000000001/8" : { "interfaces" : [ { "ips" : [ "1000::3ff/120" ], "vlan-untagged": 21 } ] } }'
         lib.moveHost( main, "h1v6", "leaf1", "leaf1", "1000::3ff", prefixLen=128, cfg=h1v6cfg, ipv6=True )
         hostLocations = { "h1v6": "of:0000000000000001/8" }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
-
         # FIXME: We don't have any tagged hosts on DAAS-1
 
+        # Move an untagged IPv4 host on DAAS-2
         h13v4cfg = '{"of:0000000000000006/7" : { "interfaces" : [ { "ips" : [ "10.5.20.254/24" ], "vlan-untagged": 20 } ] } }'
         lib.moveHost( main, "h13v4", "leaf6", "leaf6", "10.5.20.254", prefixLen=24, cfg=h13v4cfg )
         hostLocations = { "h13v4": "of:0000000000000006/7" }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
-
+        # Move an untagged IPv6 host on DAAS-2
         h13v6cfg = '{"of:0000000000000006/8" : { "interfaces" : [ { "ips" : [ "1012::3ff/120" ], "vlan-untagged": 26 } ] } }'
         lib.moveHost( main, "h13v6", "leaf6", "leaf6", "1012::3ff", prefixLen=128, cfg=h13v6cfg, ipv6=True )
         hostLocations = { "h13v6": "of:0000000000000006/8" }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
-
+        # Move a tagged IPv4 host on DAAS-2
         h12v4cfg = '{"of:0000000000000006/9" : { "interfaces" : [ { "ips" : [ "10.5.10.254/24" ], "vlan-tagged": [80] } ] } }'
         lib.moveHost( main, "h12v4", "leaf6", "leaf6", "10.5.10.254", prefixLen=24, cfg=h12v4cfg, vlan=80 )
         hostLocations = { "h12v4": "of:0000000000000006/9" }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
-
         # FIXME: Due to CORD-3079, we are not able to test movement of tagged IPv6 hosts at the moment
         '''
         h12v6cfg = '{"of:0000000000000006/10" : { "interfaces" : [ { "ips" : [ "1011::3ff/120" ], "vlan-tagged": [127] } ] } }'
@@ -1357,28 +1356,42 @@
         time.sleep( float( main.params[ 'timers' ][ 'balanceMasterSleep' ] ) )
         verify( main )
 
+        # Move an untagged IPv4 host
+        lib.addStaticOnosRoute( main, "10.2.31.0/24", "10.2.30.1" )
+        lib.startScapyHosts( main, scapyNames=[ 'h4v4Scapy' ], mininetNames=[ 'h4v4' ] )
+        lib.verifyTraffic( main, main.internalIpv4Hosts, "10.2.31.1", "h4v4Scapy", "h4v4-bond0" )
         h4v4cfg = '''{"of:0000000000000002/12" : { "interfaces" : [ { "ips" : [ "10.2.30.254/24" ], "vlan-untagged": 16 } ] },
                       "of:0000000000000003/14" : { "interfaces" : [ { "ips" : [ "10.2.30.254/24" ], "vlan-untagged": 16 } ] } }'''
         lib.moveDualHomedHost( main, "h4v4", "leaf2", "leaf3", "leaf2", "leaf3", "10.2.30.254", prefixLen=24, cfg=h4v4cfg )
         hostLocations = { "h4v4": [ "of:0000000000000002/12", "of:0000000000000003/14" ] }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
+        lib.verifyTraffic( main, main.internalIpv4Hosts, "10.2.31.1", "h4v4Scapy", "h4v4-bond1" )
 
+        # Move an untagged IPv6 host
+        lib.addStaticOnosRoute( main, "1003::400/120", "1003::3fe" )
+        lib.startScapyHosts( main, scapyNames=[ 'h4v6Scapy' ], mininetNames=[ 'h4v6' ] )
+        lib.verifyTraffic( main, main.internalIpv6Hosts, "1003::4fe", "h4v6Scapy", "h4v6-bond0", ipv6=True )
         h4v6cfg = '''{"of:0000000000000002/13" : { "interfaces" : [ { "ips" : [ "1003::3ff/120" ], "vlan-untagged": 24 } ] },
                       "of:0000000000000003/15" : { "interfaces" : [ { "ips" : [ "1003::3ff/120" ], "vlan-untagged": 24 } ] } }'''
         lib.moveDualHomedHost( main, "h4v6", "leaf2", "leaf3", "leaf2", "leaf3", "1003::3fe", prefixLen=128, cfg=h4v6cfg, ipv6=True )
         hostLocations = { "h4v6": [ "of:0000000000000002/13", "of:0000000000000003/15" ] }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
+        lib.verifyTraffic( main, main.internalIpv6Hosts, "1003::4fe", "h4v6Scapy", "h4v6-bond1", ipv6=True )
 
+        # Move a tagged IPv4 host
+        lib.addStaticOnosRoute( main, "10.2.21.0/24", "10.2.20.1" )
+        lib.startScapyHosts( main, scapyNames=[ 'h5v4Scapy' ], mininetNames=[ 'h5v4' ] )
+        lib.verifyTraffic( main, main.internalIpv4Hosts, "10.2.21.1", "h5v4Scapy", "h5v4-bond0" )
         h5v4cfg = '''{"of:0000000000000002/14" : { "interfaces" : [ { "ips" : [ "10.2.20.254/24" ], "vlan-tagged": [30] } ] },
                       "of:0000000000000003/16" : { "interfaces" : [ { "ips" : [ "10.2.20.254/24" ], "vlan-tagged": [30] } ] } }'''
         lib.moveDualHomedHost( main, "h5v4", "leaf2", "leaf3", "leaf2", "leaf3", "10.2.20.254", prefixLen=24, cfg=h5v4cfg, vlan=30 )
         hostLocations = { "h5v4": [ "of:0000000000000002/14", "of:0000000000000003/16" ] }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
+        lib.verifyTraffic( main, main.internalIpv4Hosts, "10.2.21.1", "h5v4Scapy", "h5v4-bond1" )
 
-        # TODO: test static routes that point to the moved host
         lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
 
     def CASE653( self, main ):
@@ -1396,26 +1409,40 @@
         time.sleep( float( main.params[ 'timers' ][ 'balanceMasterSleep' ] ) )
         verify( main )
 
+        # Move an untagged IPv4 host
+        lib.addStaticOnosRoute( main, "10.2.31.0/24", "10.2.30.1" )
+        lib.startScapyHosts( main, scapyNames=[ 'h4v4Scapy' ], mininetNames=[ 'h4v4' ] )
+        lib.verifyTraffic( main, main.internalIpv4Hosts, "10.2.31.1", "h4v4Scapy", "h4v4-bond0" )
         h4v4cfg = '''{"of:0000000000000002/12" : { "interfaces" : [ { "ips" : [ "10.2.30.254/24" ], "vlan-untagged": 16 } ] },
                       "of:0000000000000003/14" : { "interfaces" : [ { "ips" : [ "10.2.30.254/24" ], "vlan-untagged": 16 } ] } }'''
         lib.moveDualHomedHost( main, "h4v4", "leaf2", "leaf3", "leaf2", "leaf3", "10.2.30.254", macAddr="00:aa:01:00:00:03", prefixLen=24, cfg=h4v4cfg )
         hostLocations = { "h4v4": [ "of:0000000000000002/12", "of:0000000000000003/14" ] }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
+        lib.verifyTraffic( main, main.internalIpv4Hosts, "10.2.31.1", "h4v4Scapy", "h4v4-bond1" )
 
+        # Move an untagged IPv6 host
+        lib.addStaticOnosRoute( main, "1003::400/120", "1003::3fe" )
+        lib.startScapyHosts( main, scapyNames=[ 'h4v6Scapy' ], mininetNames=[ 'h4v6' ] )
+        lib.verifyTraffic( main, main.internalIpv6Hosts, "1003::4fe", "h4v6Scapy", "h4v6-bond0", ipv6=True )
         h4v6cfg = '''{"of:0000000000000002/13" : { "interfaces" : [ { "ips" : [ "1003::3ff/120" ], "vlan-untagged": 24 } ] },
                       "of:0000000000000003/15" : { "interfaces" : [ { "ips" : [ "1003::3ff/120" ], "vlan-untagged": 24 } ] } }'''
         lib.moveDualHomedHost( main, "h4v6", "leaf2", "leaf3", "leaf2", "leaf3", "1003::3fe", macAddr="00:bb:01:00:00:03", prefixLen=128, cfg=h4v6cfg, ipv6=True )
         hostLocations = { "h4v6": [ "of:0000000000000002/13", "of:0000000000000003/15" ] }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
+        lib.verifyTraffic( main, main.internalIpv6Hosts, "1003::4fe", "h4v6Scapy", "h4v6-bond1", ipv6=True )
 
+        # Move a tagged IPv4 host
+        lib.addStaticOnosRoute( main, "10.2.21.0/24", "10.2.20.1" )
+        lib.startScapyHosts( main, scapyNames=[ 'h5v4Scapy' ], mininetNames=[ 'h5v4' ] )
+        lib.verifyTraffic( main, main.internalIpv4Hosts, "10.2.21.1", "h5v4Scapy", "h5v4-bond0" )
         h5v4cfg = '''{"of:0000000000000002/14" : { "interfaces" : [ { "ips" : [ "10.2.20.254/24" ], "vlan-tagged": [30] } ] },
                       "of:0000000000000003/16" : { "interfaces" : [ { "ips" : [ "10.2.20.254/24" ], "vlan-tagged": [30] } ] } }'''
         lib.moveDualHomedHost( main, "h5v4", "leaf2", "leaf3", "leaf2", "leaf3", "10.2.20.254", macAddr="00:aa:01:00:00:04", prefixLen=24, cfg=h5v4cfg, vlan=30 )
         hostLocations = { "h5v4": [ "of:0000000000000002/14", "of:0000000000000003/16" ] }
         lib.verifyHostLocations( main, hostLocations )
         verify( main )
+        lib.verifyTraffic( main, main.internalIpv4Hosts, "10.2.21.1", "h5v4Scapy", "h5v4-bond1" )
 
-        # TODO: test static routes that point to the moved host
         lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo
index dbb415d..fd61ee9 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo
@@ -32,5 +32,15 @@
             </COMPONENTS>
         </Mininet1>
 
+        <Scapy>
+            <host>OCN</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>MininetScapyCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </Scapy>
     </COMPONENT>
 </TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 8c6e22a..5bd3bdd 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -1102,23 +1102,71 @@
         main.Cluster.active( 0 ).REST.setNetCfg( json.loads( json.dumps( cfg ) ) )
 
     @staticmethod
-    def startScapyHosts( main ):
+    def startScapyHosts( main, scapyNames=[], mininetNames=[] ):
         """
         Create host components and start Scapy CLIs
+        scapyNames: list of names that will be used as component names for scapy hosts
+        mininetNames: used when scapy host names are different from the host names
+        in Mininet. E.g. when scapyNames=['h1Scapy'], it's required to specify the
+        name of the corresponding Mininet host by mininetNames=['h1']
         """
         main.step( "Start Scapy CLIs" )
-        main.scapyHostNames = main.params[ 'SCAPY' ][ 'HOSTNAMES' ].split( ',' )
+        if scapyNames:
+            main.scapyNames = scapyNames
+        else:
+            main.scapyNames = main.params[ 'SCAPY' ][ 'HOSTNAMES' ].split( ',' )
         main.scapyHosts = []
-        for hostName in main.scapyHostNames:
-            main.Scapy.createHostComponent( hostName )
-            main.scapyHosts.append( getattr( main, hostName ) )
-        for host in main.scapyHosts:
-            host.startHostCli()
-            host.startScapy()
-            host.updateSelf()
-            main.log.debug( host.name )
-            main.log.debug( host.hostIp )
-            main.log.debug( host.hostMac )
+        for scapyName in main.scapyNames:
+            main.Scapy.createHostComponent( scapyName )
+            scapyHandle = getattr( main, scapyName )
+            main.scapyHosts.append( scapyHandle )
+            if mininetNames:
+                mininetName = mininetNames[ scapyNames.index( scapyName ) ]
+            else:
+                mininetName = None
+            scapyHandle.startHostCli( mininetName )
+            scapyHandle.startScapy()
+            scapyHandle.updateSelf()
+            main.log.debug( scapyHandle.name )
+            main.log.debug( scapyHandle.hostIp )
+            main.log.debug( scapyHandle.hostMac )
+
+    @staticmethod
+    def verifyTraffic( main, srcHosts, dstIp, dstHost, dstIntf, ipv6=False, expect=True, skipOnFail=True, maxRetry=2 ):
+        """
+        Verify unicast traffic by pinging from source hosts to the destination IP
+        and capturing the packets at the destination host using Scapy.
+        srcHosts: List of host names to send the ping packets
+        dstIp: destination IP of the ping packets
+        dstHost: host that runs Scapy to capture the packets
+        dstIntf: name of the interface on the destination host
+        expect: use True if the ping is expected to be captured at destination;
+                Otherwise False
+        skipOnFail: skip the rest of this test case if result is not expected
+        maxRetry: number of retries allowed
+        """
+        from tests.dependencies.topology import Topology
+        try:
+            main.topo
+        except ( NameError, AttributeError ):
+            main.topo = Topology()
+        main.step( "Verify traffic to {} by capturing packets on {}".format( dstIp, dstHost ) )
+        result = main.TRUE
+        for srcHost in srcHosts:
+            trafficResult = main.topo.pingAndCapture( srcHost, dstIp, dstHost, dstIntf, ipv6,
+                                                      expect, maxRetry, True )
+            if not trafficResult:
+                main.stop()
+                result = main.FALSE
+                main.log.warn( "Scapy result from {} to {} is not as expected".format( srcHost, dstIp ) )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=result,
+                                 onpass="Verify traffic to {}: Pass".format( dstIp ),
+                                 onfail="Verify traffic to {}: Fail".format( dstIp ) )
+        if skipOnFail and result != main.TRUE:
+            Testcaselib.saveOnosDiagnostics( main )
+            Testcaselib.cleanup( main, copyKarafLog=False )
+            main.skipCase()
 
     @staticmethod
     def verifyMulticastTraffic( main, routeName, expect, skipOnFail=True, maxRetry=1 ):
diff --git a/TestON/tests/dependencies/topology.py b/TestON/tests/dependencies/topology.py
index e9fe8d7..cbb14e7 100644
--- a/TestON/tests/dependencies/topology.py
+++ b/TestON/tests/dependencies/topology.py
@@ -308,9 +308,8 @@
     def sendScapyPackets( self, sender, receiver, pktFilter, pkt, sIface=None, dIface=None, expect=True, acceptableFailed=0, collectT3=True, t3Command="" ):
         """
         Description:
-            Call sendScapyPacket and retry if neccessary
+            Send Scapy packets from sender to receiver and verify if result is as expected and retry if neccessary
             If collectT3 is True and t3Command is specified, collect t3-troubleshoot output on unexpected scapy results
-            Buiid packets on the sender side by calling functions in Scapy CLI driver
         Options:
             sender: the component of the host that is sending packets
             receiver: the component of the host that is receiving packets
@@ -321,10 +320,11 @@
             collectT3: save t3-troubleshoot output for unexpected scapy results
         Returns:
             main.TRUE if scapy result is expected, otherwise main.FALSE
+        Note: It is assumed that Scapy is already started on the destination host
         """
         main.log.info( "Sending scapy packets from  {} to {}, expected result is {}".format( sender.name, receiver.name,
                                                                                              "pass" if expect else "fail" ) )
-        scapyResult = utilities.retry( self.sendScapyPacket,
+        scapyResult = utilities.retry( self.sendScapyPacketsHelper,
                                        main.FALSE,
                                        args=( sender, receiver, pktFilter, pkt,
                                               sIface, dIface, expect ),
@@ -337,7 +337,7 @@
                                         "t3-CASE{}-{}-{}-".format( main.CurrentTestCaseNumber, sender.name, receiver.name ) )
         return scapyResult
 
-    def sendScapyPacket( self, sender, receiver, pktFilter, pkt, sIface=None, dIface=None, expect=True ):
+    def sendScapyPacketsHelper( self, sender, receiver, pktFilter, pkt, sIface=None, dIface=None, expect=True ):
         """
         Description:
             Send Scapy packets from sender to receiver and verify if result is as expected
@@ -366,3 +366,83 @@
             main.log.debug( sender.handle.before )
         packetCaptured = True if pkt in packet else False
         return main.TRUE if packetCaptured == expect else main.FALSE
+
+    def pingAndCapture( self, srcHost, dstIp, dstHost, dstIntf, ipv6=False, expect=True, acceptableFailed=0, collectT3=True, t3Simple=False ):
+        """
+        Description:
+            Ping from src host to dst IP and capture packets at dst Host using Scapy and retry if neccessary
+            If collectT3 is True, collect t3-troubleshoot output on unexpected scapy results
+        Options:
+            srcHost: name of the source host
+            dstIp: destination IP of the ping packets
+            dstHost: host that runs Scapy to capture the packets
+            dstIntf: name of the interface on the destination host
+            ipv6: ping with IPv6 if True; Otherwise IPv4
+            expect: use True if the ping is expected to be captured at destination;
+                    Otherwise False
+            acceptableFailed: maximum number of failed pings acceptable
+            collectT3: save t3-troubleshoot output for src and dst host that failed to ping
+            t3Simple: use t3-troubleshoot-simple command when collecting t3 output
+        Returns:
+            main.TRUE if packet capturing result is expected, otherwise main.FALSE
+        Note: It is assumed that Scapy is already started on the destination host
+        """
+        main.log.info( "Pinging from {} to {}, expected {} capture the packet at {}".format( srcHost, dstIp,
+                       "to" if expect else "not to", dstHost ) )
+        # Verify host component has been created
+        if not hasattr( main, srcHost ):
+            main.log.info( "Creating component for host {}".format( srcHost ) )
+            main.Network.createHostComponent( srcHost )
+            srcHandle = getattr( main, srcHost )
+            main.log.info( "Starting CLI on host {}".format( srcHost ) )
+            srcHandle.startHostCli()
+        trafficResult = utilities.retry( self.pingAndCaptureHelper,
+                                         main.FALSE,
+                                         args=( srcHost, dstIp, dstHost, dstIntf, ipv6, expect ),
+                                         attempts=acceptableFailed + 1,
+                                         sleep=1 )
+        if not trafficResult and collectT3:
+            srcIp = main.Network.getIPAddress( srcHost, proto='IPV6' if ipv6 else 'IPV4' )
+            main.log.debug( "Collecting t3 with source {} and destination {}".format( srcIp, dstIp ) )
+            cmdList = main.Cluster.active( 0 ).CLI.composeT3Command( srcIp, dstIp, ipv6, True, t3Simple )
+            if not cmdList:
+                main.log.warn( "Failed to compose T3 command with source {} and destination {}".format( srcIp, dstIp ) )
+            for i in range( 0, len( cmdList ) ):
+                cmd = cmdList[ i ]
+                main.log.debug( "t3 command: {}".format( cmd ) )
+                main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, cmd, main.logdir,
+                                            "t3-CASE{}-{}-{}-route{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp, i ),
+                                            timeout=10 )
+        return trafficResult
+
+    def pingAndCaptureHelper( self, srcHost, dstIp, dstHost, dstIntf, ipv6=False, expect=True ):
+        """
+        Description:
+            Ping from src host to dst IP and capture packets at dst Host using Scapy
+        Options:
+            srcHost: name of the source host
+            dstIp: destination IP of the ping packets
+            dstHost: host that runs Scapy to capture the packets
+            dstIntf: name of the interface on the destination host
+            ipv6: ping with IPv6 if True; Otherwise IPv4
+            expect: use True if the ping is expected to be captured at destination;
+                    Otherwise False
+        Returns:
+            main.TRUE if packet capturing result is expected, otherwise main.FALSE
+        """
+        packetCaptured = True
+        srcHandle = getattr( main, srcHost )
+        dstHandle = getattr( main, dstHost )
+        dstHandle.startFilter( ifaceName=dstIntf, pktFilter="ip host {}".format( dstIp ) )
+        srcHandle.pingHostSetAlternative( [ dstIp ], IPv6=ipv6 )
+        finished = dstHandle.checkFilter()
+        packet = ""
+        if finished:
+            packets = dstHandle.readPackets()
+            for packet in packets.splitlines():
+                main.log.debug( packet )
+        else:
+            kill = dstHandle.killFilter()
+            main.log.debug( kill )
+        packetCaptured = True if "dst={}".format( dstIp ) in packet else False
+        return main.TRUE if packetCaptured == expect else main.FALSE