[SDFAB-493] UP4 data plane failure test

Change-Id: I25a7c728d1a6d068ac5f9bb02319e6c32623a6b0
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/up4.py b/TestON/tests/USECASE/SegmentRouting/dependencies/up4.py
index bc6a49a..0f6c8a8 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/up4.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/up4.py
@@ -1,4 +1,5 @@
 from distutils.util import strtobool
+import copy
 
 FALSE = '0'
 TRUE = '1'
@@ -20,16 +21,28 @@
     """
     Utility that manages interaction with UP4 via a P4RuntimeCliDriver available
     in the cluster. Additionally, can verify connectivity by crafting GTP packets
-    via Scapy with an HostDriver component, specified via <enodeb_host>, <pdn_host>,
+    via Scapy with an HostDriver component, specified via <enodebs>, <pdn_host>,
     and <router_mac> parameters.
 
     Example params file:
     <UP4>
         <pdn_host>Compute1</pdn_host> # Needed to verify connectivity with scapy
-        <enodeb_host>Compute3</enodeb_host> # Needed to verify connectivity with scapy
+         <enodebs> # List of emulated eNodeBs
+            <enode_1>
+                <host>Compute1</host>  # Host that emulates this eNodeB
+                <interface>eno3</interface> # Name of the linux interface to use on the host, if not specified take the default
+                <enb_address>10.32.11.122</enb_address> # IP address of the eNodeB
+                <ues>ue3</ues> # Emulated ues connected to this eNB
+            </enode_1>
+            <enodeb_2>
+                <host>Compute3</host>
+                <enb_address>10.32.11.194</enb_address>
+                <ues>ue1,ue2</ues>
+            </enodeb_2>
+        </enodebs>
+        <enodeb_host>Compute3</enodeb_host>
         <router_mac>00:00:0A:4C:1C:46</router_mac> # Needed to verify connectivity with scapy
         <s1u_address>10.32.11.126</s1u_address>
-        <enb_address>10.32.11.100</enb_address>
         <ues>
             <ue2>
                 <pfcp_session_id>100</pfcp_session_id>
@@ -41,19 +54,20 @@
                 <five_g>False</five_g>
             </ue2>
         </ues>
+        <switch_to_kill>Leaf2</switch_to_kill> # Component name of the switch to kill in CASE 5
+        <enodebs_fail>enodeb_1</enodebs_fail> # List of eNodeBs that should fail traffic forwarding in CASE 5
     </UP4>
     """
 
     def __init__(self):
         self.s1u_address = None
-        self.enb_address = None
-        self.enodeb_host = None
-        self.enodeb_interface = None
+        self.enodebs = None
         self.pdn_host = None
         self.pdn_interface = None
         self.router_mac = None
-        self.emulated_ues = []
+        self.emulated_ues = {}
         self.up4_client = None
+        self.no_host = False
 
     def setup(self, p4rt_client, no_host=False):
         """
@@ -63,58 +77,73 @@
         :return:
         """
         self.s1u_address = main.params["UP4"]["s1u_address"]
-        self.enb_address = main.params["UP4"]["enb_address"]
         self.emulated_ues = main.params["UP4"]['ues']
         self.up4_client = p4rt_client
+        self.no_host = no_host
 
         # Optional Parameters
-        if not no_host:
-            if "enodeb_host" in main.params["UP4"]:
-                self.enodeb_host = getattr(main,
-                                           main.params["UP4"]["enodeb_host"])
-                self.enodeb_interface = self.enodeb_host.interfaces[0]
-            if "pdn_host" in main.params["UP4"]:
-                self.pdn_host = getattr(main, main.params["UP4"]["pdn_host"])
-                self.pdn_interface = self.pdn_host.interfaces[0]
-            self.router_mac = main.params["UP4"].get("router_mac", None)
+
+        self.enodebs = copy.deepcopy((main.params["UP4"]["enodebs"]))
+        for enb in self.enodebs.values():
+            enb["ues"] = enb["ues"].split(",")
+            enb["host"] = getattr(main, enb["host"])
+            # If interface not provided by the params, use the default in the host
+            if "interface" not in enb.keys():
+                enb["interface"] = enb["host"].interfaces[0]["name"]
+        if "pdn_host" in main.params["UP4"]:
+            self.pdn_host = getattr(main, main.params["UP4"]["pdn_host"])
+            self.pdn_interface = self.pdn_host.interfaces[0]
+        self.router_mac = main.params["UP4"].get("router_mac", None)
 
         # Start components
         self.up4_client.startP4RtClient()
-        if self.enodeb_host is not None:
-            self.enodeb_host.startScapy(ifaceName=self.enodeb_interface["name"],
-                                        enableGtp=True)
-        if self.pdn_host is not None:
-            self.pdn_host.startScapy(ifaceName=self.pdn_interface["name"])
+        if not self.no_host:
+            if self.enodebs is not None:
+                for enb in self.enodebs.values():
+                    enb["host"].startScapy(ifaceName=enb["interface"],
+                                            enableGtp=True)
+            if self.pdn_host is not None:
+                self.pdn_host.startScapy(ifaceName=self.pdn_interface["name"])
 
     def teardown(self):
         self.up4_client.stopP4RtClient()
-        if self.enodeb_host is not None:
-            self.enodeb_host.stopScapy()
-        if self.pdn_host is not None:
-            self.pdn_host.stopScapy()
+        if not self.no_host:
+            if self.enodebs is not None:
+                for enb in self.enodebs.values():
+                    enb["host"].stopScapy()
+            if self.pdn_host is not None:
+                self.pdn_host.stopScapy()
 
     def attachUes(self):
-        for ue in self.emulated_ues.values():
+        for (name, ue) in self.emulated_ues.items():
             ue = UP4.__sanitizeUeData(ue)
-            self.attachUe(**ue)
+            self.attachUe(name, **ue)
 
     def detachUes(self):
-        for ue in self.emulated_ues.values():
+        for (name, ue) in self.emulated_ues.items():
             ue = UP4.__sanitizeUeData(ue)
-            self.detachUe(**ue)
+            self.detachUe(name, **ue)
 
-    def testUpstreamTraffic(self):
-        if self.enodeb_host is None or self.pdn_host is None:
+    def testUpstreamTraffic(self, enb_names=None, shouldFail=False):
+        if self.enodebs is None or self.pdn_host is None:
             main.log.error(
                 "Need eNodeB and PDN host params to generate scapy traffic")
             return
         # Scapy filter needs to start before sending traffic
+        if enb_names is None or enb_names == []:
+            enodebs = self.enodebs.values()
+        else:
+            enodebs = [self.enodebs[enb] for enb in enb_names]
         pkt_filter_upstream = ""
-        for ue in self.emulated_ues.values():
-            if "ue_address" in ue:
-                if len(pkt_filter_upstream) != 0:
-                    pkt_filter_upstream += " or "
-                pkt_filter_upstream += "src host " + ue["ue_address"]
+        ues = []
+        for enb in enodebs:
+            for ue_name in enb["ues"]:
+                ue = self.emulated_ues[ue_name]
+                if "ue_address" in ue:
+                    ues.append(ue)
+                    if len(pkt_filter_upstream) != 0:
+                        pkt_filter_upstream += " or "
+                    pkt_filter_upstream += "src host " + ue["ue_address"]
         pkt_filter_upstream = "ip and udp dst port %s and (%s) and dst host %s" % \
                               (PDN_PORT, pkt_filter_upstream,
                                self.pdn_interface["ips"][0])
@@ -122,33 +151,35 @@
                       (self.pdn_host.name, self.pdn_interface["name"]))
         main.log.debug("BPF Filter Upstream: \n %s" % pkt_filter_upstream)
         self.pdn_host.startFilter(ifaceName=self.pdn_interface["name"],
-                                  sniffCount=len(self.emulated_ues),
+                                  sniffCount=len(ues),
                                   pktFilter=pkt_filter_upstream)
 
         main.log.info(
-            "Sending %d packets from eNodeB host" % len(self.emulated_ues))
-        for ue in self.emulated_ues.values():
-            UP4.buildGtpPacket(self.enodeb_host,
-                               src_ip_outer=self.enb_address,
-                               dst_ip_outer=self.s1u_address,
-                               src_ip_inner=ue["ue_address"],
-                               dst_ip_inner=self.pdn_interface["ips"][0],
-                               src_udp_inner=UE_PORT,
-                               dst_udp_inner=PDN_PORT,
-                               teid=int(ue["teid"]))
-
-            self.enodeb_host.sendPacket(iface=self.enodeb_interface["name"])
+            "Sending %d packets from eNodeB host" % len(ues))
+        for enb in enodebs:
+            for ue_name in enb["ues"]:
+                main.log.info(ue_name)
+                ue = self.emulated_ues[ue_name]
+                main.log.info(str(ue))
+                UP4.buildGtpPacket(enb["host"],
+                                   src_ip_outer=enb["enb_address"],
+                                   dst_ip_outer=self.s1u_address,
+                                   src_ip_inner=ue["ue_address"],
+                                   dst_ip_inner=self.pdn_interface["ips"][0],
+                                   src_udp_inner=UE_PORT,
+                                   dst_udp_inner=PDN_PORT,
+                                   teid=int(ue["teid"]))
+                enb["host"].sendPacket(iface=enb["interface"])
 
         packets = UP4.checkFilterAndGetPackets(self.pdn_host)
         fail = False
-        if len(self.emulated_ues) != packets.count('Ether'):
+        if len(ues) != packets.count('Ether'):
             fail = True
             msg = "Failed to capture packets in PDN.\n" + str(packets)
         else:
             msg = "Correctly captured packet in PDN. "
         # We expect exactly 1 packet per UE
-        pktsFiltered = [packets.count("src=" + ue["ue_address"])
-                        for ue in self.emulated_ues.values()]
+        pktsFiltered = [packets.count("src=" + ue["ue_address"]) for ue in ues]
         if pktsFiltered.count(1) != len(pktsFiltered):
             fail = True
             msg += "\nError on the number of packets per UE in downstream.\n" + str(packets)
@@ -156,25 +187,33 @@
             msg += "\nOne packet per UE in upstream. "
 
         utilities.assert_equal(
-            expect=False, actual=fail, onpass=msg, onfail=msg)
+            expect=shouldFail, actual=fail, onpass=msg, onfail=msg)
 
-    def testDownstreamTraffic(self):
-        if self.enodeb_host is None or self.pdn_host is None:
+    def testDownstreamTraffic(self, enb_names=None, shouldFail=False):
+        if self.enodebs is None or self.pdn_host is None:
             main.log.error(
                 "Need eNodeB and PDN host params to generate scapy traffic")
             return
-        pkt_filter_downstream = "ip and udp src port %d and udp dst port %d and dst host %s and src host %s" % (
-            GPDU_PORT, GPDU_PORT, self.enb_address, self.s1u_address)
-        main.log.info("Start listening on %s intf %s" % (
-            self.enodeb_host.name, self.enodeb_interface["name"]))
-        main.log.debug("BPF Filter Downstream: \n %s" % pkt_filter_downstream)
-        self.enodeb_host.startFilter(ifaceName=self.enodeb_interface["name"],
-                                     sniffCount=len(self.emulated_ues),
-                                     pktFilter=pkt_filter_downstream)
+        if enb_names is None or enb_names == []:
+            enodebs = self.enodebs.values()
+        else:
+            enodebs = [self.enodebs[enb] for enb in enb_names]
+        pkt_filter_downstream = "ip and udp src port %d and udp dst port %d and src host %s" % (
+            GPDU_PORT, GPDU_PORT, self.s1u_address)
+        ues = []
+        for enb in enodebs:
+            filter_down = pkt_filter_downstream + " and dst host %s" % enb["enb_address"]
+            main.log.info("Start listening on %s intf %s" % (
+                enb["host"], enb["interface"]))
+            main.log.debug("BPF Filter Downstream: \n %s" % filter_down)
+            enb["host"].startFilter(ifaceName=enb["interface"],
+                                    sniffCount=len(enb["ues"]),
+                                    pktFilter=filter_down)
+            ues.extend([self.emulated_ues[ue_name] for ue_name in enb["ues"]])
 
         main.log.info(
-            "Sending %d packets from PDN host" % len(self.emulated_ues))
-        for ue in self.emulated_ues.values():
+            "Sending %d packets from PDN host" % len(ues))
+        for ue in ues:
             # From PDN we have to set dest MAC, otherwise scapy will do ARP
             # request for the UE IP address.
             UP4.buildUdpPacket(self.pdn_host,
@@ -184,19 +223,21 @@
                                src_udp=PDN_PORT,
                                dst_udp=UE_PORT)
             self.pdn_host.sendPacket(iface=self.pdn_interface["name"])
-
-        packets = UP4.checkFilterAndGetPackets(self.enodeb_host)
-
+        packets = ""
+        for enb in enodebs:
+            pkt = UP4.checkFilterAndGetPackets(enb["host"])
+            packets += pkt
         # The BPF filter might capture non-GTP packets because we can't filter
         # GTP header in BPF. For this reason, check that the captured packets
         # are from the expected tunnels.
         # TODO: check inner UDP and IP fields as well
         # FIXME: with newer scapy TEID becomes teid (required for Scapy 2.4.5)
-        pktsFiltered = [packets.count("TEID=" + hex(int(ue["teid"])) + "L ")
-                        for ue in self.emulated_ues.values()]
-
+        pktsFiltered= [packets.count("TEID=" + hex(int(ue["teid"])) + "L ")
+             for ue in ues]
+        main.log.info("PACKETS: " + str(packets))
+        main.log.info("PKTs Filtered: " + str(pktsFiltered))
         fail = False
-        if len(self.emulated_ues) != sum(pktsFiltered):
+        if len(ues) != sum(pktsFiltered):
             fail = True
             msg = "Failed to capture packets in eNodeB.\n" + str(packets)
         else:
@@ -209,7 +250,7 @@
             msg += "\nOne packet per GTP TEID in downstream. "
 
         utilities.assert_equal(
-            expect=False, actual=fail, onpass=msg, onfail=msg)
+            expect=shouldFail, actual=fail, onpass=msg, onfail=msg)
 
     def readPdrsNumber(self):
         """
@@ -305,7 +346,7 @@
         fars = onosCli.sendline(cmdStr="up4:read-fars", showResponse=True,
                                 noExit=True, expectJson=False)
         fail = False
-        for ue in self.emulated_ues.values():
+        for (ue_name, ue) in self.emulated_ues.items():
             if pdrs.count(self.upPdrOnosString(**ue)) != 1:
                 failMsg += self.upPdrOnosString(**ue) + "\n"
                 fail = True
@@ -315,8 +356,8 @@
             if fars.count(self.upFarOnosString(**ue)) != 1:
                 failMsg += self.upFarOnosString(**ue) + "\n"
                 fail = True
-            if fars.count(self.downFarOnosString(**ue)) != 1:
-                failMsg += self.downFarOnosString(**ue) + "\n"
+            if fars.count(self.downFarOnosString(ue_name, **ue)) != 1:
+                failMsg += self.downFarOnosString(ue_name, **ue) + "\n"
                 fail = True
         return not fail
 
@@ -351,16 +392,16 @@
         return "PDR{{Match(Dst={}, !GTP) -> LoadParams(SEID={}, FAR={}, CtrIdx={})}}".format(
             ue_address, hex(int(pfcp_session_id)), far_id_down, ctr_id_down)
 
-    def downFarOnosString(self, pfcp_session_id, teid=None, down_id=None,
+    def downFarOnosString(self, ue_name, pfcp_session_id, teid=None, down_id=None,
                           teid_down=None, far_id_down=None, **kwargs):
         if down_id is not None:
             far_id_down = down_id
         if teid is not None:
             teid_down = teid
+        enb_address = self.__getEnbAddress(ue_name)
         return "FAR{{Match(ID={}, SEID={}) -> Encap(Src={}, SPort={}, TEID={}, Dst={})}}".format(
             far_id_down, hex(int(pfcp_session_id)), self.s1u_address, GPDU_PORT,
-            hex(int(teid_down)),
-            self.enb_address)
+            hex(int(teid_down)), enb_address)
 
     def upFarOnosString(self, pfcp_session_id, up_id=None, far_id_up=None,
                         **kwargs):
@@ -377,13 +418,14 @@
             ue["qfi"] = None
         return ue
 
-    def attachUe(self, pfcp_session_id, ue_address,
+    def attachUe(self, ue_name, pfcp_session_id, ue_address,
                  teid=None, up_id=None, down_id=None,
                  teid_up=None, teid_down=None,
                  pdr_id_up=None, far_id_up=None, ctr_id_up=None,
                  pdr_id_down=None, far_id_down=None, ctr_id_down=None,
                  qfi=None, five_g=False):
-        self.__programUp4Rules(pfcp_session_id,
+        self.__programUp4Rules(ue_name,
+                               pfcp_session_id,
                                ue_address,
                                teid, up_id, down_id,
                                teid_up, teid_down,
@@ -391,13 +433,14 @@
                                pdr_id_down, far_id_down, ctr_id_down,
                                qfi, five_g, action="program")
 
-    def detachUe(self, pfcp_session_id, ue_address,
+    def detachUe(self, ue_name, pfcp_session_id, ue_address,
                  teid=None, up_id=None, down_id=None,
                  teid_up=None, teid_down=None,
                  pdr_id_up=None, far_id_up=None, ctr_id_up=None,
                  pdr_id_down=None, far_id_down=None, ctr_id_down=None,
                  qfi=None, five_g=False):
-        self.__programUp4Rules(pfcp_session_id,
+        self.__programUp4Rules(ue_name,
+                               pfcp_session_id,
                                ue_address,
                                teid, up_id, down_id,
                                teid_up, teid_down,
@@ -405,7 +448,7 @@
                                pdr_id_down, far_id_down, ctr_id_down,
                                qfi, five_g, action="clear")
 
-    def __programUp4Rules(self, pfcp_session_id, ue_address,
+    def __programUp4Rules(self, ue_name, pfcp_session_id, ue_address,
                           teid=None, up_id=None, down_id=None,
                           teid_up=None, teid_down=None,
                           pdr_id_up=None, far_id_up=None, ctr_id_up=None,
@@ -425,6 +468,9 @@
 
         entries = []
 
+        # Retrieve eNobeB address from eNodeB list
+        enb_address = self.__getEnbAddress(ue_name)
+
         # ========================#
         # PDR Entries
         # ========================#
@@ -518,7 +564,7 @@
         actionParams['needs_buffering'] = FALSE
         actionParams['tunnel_type'] = TUNNEL_TYPE_GPDU
         actionParams['src_addr'] = str(self.s1u_address)
-        actionParams['dst_addr'] = str(self.enb_address)
+        actionParams['dst_addr'] = str(enb_address)
         actionParams['teid'] = str(teid_down)
         actionParams['sport'] = TUNNEL_SPORT
         if not self.__add_entry(tableName, actionName, matchFields,
@@ -555,6 +601,13 @@
             else:
                 main.log.error("Error during table delete")
 
+    def __getEnbAddress(self, ue_name):
+        for enb in self.enodebs.values():
+            if ue_name in enb["ues"]:
+                return enb["enb_address"]
+        main.log.error("Missing eNodeB address!")
+        return ""
+
     @staticmethod
     def buildGtpPacket(host, src_ip_outer, dst_ip_outer, src_ip_inner,
                        dst_ip_inner, src_udp_inner, dst_udp_inner, teid):