Merge "[SDFAB-96] Use UP4 APIs from different ONOS instances"
diff --git a/TestON/drivers/common/cli/p4runtimeclidriver.py b/TestON/drivers/common/cli/p4runtimeclidriver.py
index 6728cba..e2ef0c8 100644
--- a/TestON/drivers/common/cli/p4runtimeclidriver.py
+++ b/TestON/drivers/common/cli/p4runtimeclidriver.py
@@ -293,6 +293,29 @@
             main.log.exception(self.name + ": Uncaught exception!")
             main.cleanAndExit()
 
+    def readNumberTableEntries(self, tableName):
+        """
+        Read table entries and return the number of entries present in a table.
+
+        :param tableName: Name of table to read from
+        :return: Number of entries,
+        """
+        try:
+            main.log.debug(self.name + ": Reading table entries from " + tableName)
+            cmd = 'table_entry["%s"].read(lambda te: print(te))' % tableName
+            response = self.__clearSendAndExpect(cmd, clearBufferAfter=True)
+            # Every table entries starts with "table_id: [P4RT obj ID] ("[tableName]")"
+            return response.count("table_id")
+        except pexpect.TIMEOUT:
+            main.log.exception(self.name + ": Command timed out")
+            return main.FALSE
+        except pexpect.EOF:
+            main.log.exception(self.name + ": connection closed.")
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception(self.name + ": Uncaught exception!")
+            main.cleanAndExit()
+
     def disconnect(self):
         """
         Called at the end of the test to stop the p4rt CLI component and
@@ -337,10 +360,12 @@
             response = main.FALSE
         return response
 
-    def __clearSendAndExpect(self, cmd):
-        self.clearBuffer()
+    def __clearSendAndExpect(self, cmd, clearBufferAfter=False, debug=False):
+        self.clearBuffer(debug)
         self.handle.sendline(cmd)
         self.handle.expect(self.p4rtShPrompt)
+        if clearBufferAfter:
+            return self.clearBuffer()
         return self.handle.before
 
     def clearBuffer(self, debug=False):
diff --git a/TestON/tests/USECASE/SegmentRouting/UP4/UP4.params b/TestON/tests/USECASE/SegmentRouting/UP4/UP4.params
index 98dc41e..ef8fd49 100644
--- a/TestON/tests/USECASE/SegmentRouting/UP4/UP4.params
+++ b/TestON/tests/USECASE/SegmentRouting/UP4/UP4.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>1,2</testcases>
+    <testcases>1,2,3</testcases>
 
     <GRAPH>
         <nodeCluster>pairedleaves</nodeCluster>
diff --git a/TestON/tests/USECASE/SegmentRouting/UP4/UP4.py b/TestON/tests/USECASE/SegmentRouting/UP4/UP4.py
index 14b845f..4ca5eda 100644
--- a/TestON/tests/USECASE/SegmentRouting/UP4/UP4.py
+++ b/TestON/tests/USECASE/SegmentRouting/UP4/UP4.py
@@ -3,7 +3,6 @@
     def __init__(self):
         self.default = ''
 
-    # TODO: add test case that checks entries are being inserted and deleted from ONOS correclty
     def CASE1(self, main):
         main.case("Fabric UPF traffic terminated in the fabric")
         """
@@ -250,3 +249,119 @@
         up4.teardown()
         bess_host.stopScapy()
         run.cleanup(main)
+
+    def CASE3(self, main):
+        main.case("Verify UP4 from different ONOS instances")
+        """
+        Program PDRs and FARs via UP4 on first ONOS instance
+        Repeat for all ONOS Instances:
+            Verify PDRs and FARs via P4RT
+            Disconnect P4RT client
+        Verify and delete PDRs and FARs via UP4 on the third ONOS instance
+        Repeat for all ONOS Instance:
+            Verify removed PDRs and FARs via P4RT
+            Disconnect P4RT client
+        """
+        try:
+            from tests.USECASE.SegmentRouting.dependencies.up4 import UP4
+            from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+                Testcaselib as run
+        except ImportError as e:
+            main.log.error("Import not found. Exiting the test")
+            main.log.error(e)
+            main.cleanAndExit()
+
+        run.initTest(main)
+        main.log.info(main.Cluster.numCtrls)
+        main.Cluster.setRunningNode(3)
+        run.installOnos(main, skipPackage=True, cliSleep=5)
+
+        onos_cli_0 = main.Cluster.active(0).CLI
+        onos_cli_1 = main.Cluster.active(1).CLI
+        onos_cli_2 = main.Cluster.active(2).CLI
+        up4_0 = UP4()
+        up4_1 = UP4()
+        up4_2 = UP4()
+
+        main.step("Program PDRs and FARs via UP4 on ONOS 0")
+        up4_0.setup(main.Cluster.active(0).p4rtUp4, no_host=True)
+        up4_0.attachUes()
+        up4_0.verifyUp4Flow(onos_cli_0)
+        up4_0.teardown()
+
+        main.step("Verify PDRs and FARs number via UP4 P4RT on ONOS 1")
+        up4_1.setup(main.Cluster.active(1).p4rtUp4, no_host=True)
+        utilities.assert_equal(
+            expect=True,
+            actual=up4_1.verifyUesFlowNumberP4rt(),
+            onpass="Correct number of PDRs and FARs",
+            onfail="Wrong number of PDRs and FARs"
+        )
+        up4_1.teardown()
+
+        main.step("Verify PDRs and FARs number via UP4 P4RT on ONOS 2")
+        up4_2.setup(main.Cluster.active(2).p4rtUp4, no_host=True)
+        utilities.assert_equal(
+            expect=True,
+            actual=up4_2.verifyUesFlowNumberP4rt(),
+            onpass="Correct number of PDRs and FARs",
+            onfail="Wrong number of PDRs and FARs"
+        )
+
+        main.step("Verify all ONOS instances have the same number of flows")
+        onos_0_flow_count = onos_cli_0.checkFlowCount()
+        onos_1_flow_count = onos_cli_1.checkFlowCount()
+        onos_2_flow_count = onos_cli_2.checkFlowCount()
+        utilities.assert_equal(
+            expect=True,
+            actual=onos_0_flow_count == onos_1_flow_count == onos_2_flow_count,
+            onpass="All ONOS instances have the same number of flows",
+            onfail="ONOS instances have different number of flows: (%d, %d, %d)" % (
+                onos_0_flow_count, onos_1_flow_count, onos_2_flow_count)
+        )
+
+        main.step("Remove PDRs and FARs via UP4 on ONOS 2")
+        up4_2.detachUes()
+        up4_2.verifyNoUesFlow(onos_cli_2)
+
+        main.step("Verify no PDRs and FARs via UP4 P4RT on ONOS 2")
+        utilities.assert_equal(
+            expect=True,
+            actual=up4_2.verifyNoUesFlowNumberP4rt(),
+            onpass="No PDRs and FARs",
+            onfail="Stale PDRs and FARs"
+        )
+        up4_2.teardown()
+
+        main.step("Verify no PDRs and FARs via UP4 P4RT on ONOS 1")
+        up4_1.setup(main.Cluster.active(1).p4rtUp4, no_host=True)
+        utilities.assert_equal(
+            expect=True,
+            actual=up4_1.verifyNoUesFlowNumberP4rt(),
+            onpass="No PDRs and FARs",
+            onfail="Stale PDRs and FARs"
+        )
+        up4_1.teardown()
+
+        main.step("Verify no PDRs and FARs via UP4 P4RT on ONOS 0")
+        up4_0.setup(main.Cluster.active(0).p4rtUp4, no_host=True)
+        utilities.assert_equal(
+            expect=True,
+            actual=up4_0.verifyNoUesFlowNumberP4rt(),
+            onpass="No PDRs and FARs",
+            onfail="Stale PDRs and FARs"
+        )
+        up4_0.teardown()
+
+        main.step("Verify all ONOS instances have the same number of flows")
+        onos_0_flow_count = onos_cli_0.checkFlowCount()
+        onos_1_flow_count = onos_cli_1.checkFlowCount()
+        onos_2_flow_count = onos_cli_2.checkFlowCount()
+        utilities.assert_equal(
+            expect=True,
+            actual=onos_0_flow_count == onos_1_flow_count == onos_2_flow_count,
+            onpass="All ONOS instances have the same number of flows",
+            onfail="ONOS instances have different number of flows: (%d, %d, %d)" % (
+            onos_0_flow_count, onos_1_flow_count, onos_2_flow_count)
+        )
+        run.cleanup(main)
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/up4.py b/TestON/tests/USECASE/SegmentRouting/dependencies/up4.py
index cd883e4..c66c45e 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/up4.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/up4.py
@@ -53,20 +53,28 @@
         self.emulated_ues = []
         self.up4_client = None
 
-    def setup(self, p4rt_client):
+    def setup(self, p4rt_client, no_host=False):
+        """
+        Set up P4RT and scapy on eNB and PDN hosts
+        :param p4rt_client: a P4RuntimeCliDriver component
+        :param no_host: True if you don't want to start scapy on the hosts
+        :return:
+        """
         self.s1u_address = main.params["UP4"]["s1u_address"]
         self.enb_address = main.params["UP4"]["enb_address"]
         self.emulated_ues = main.params["UP4"]['ues']
         self.up4_client = p4rt_client
 
         # Optional Parameters
-        if "enodeb_host" in main.params["UP4"]:
-            self.enodeb_host = getattr(main, main.params["UP4"]["enodeb_host"])
-            self.enodeb_interface = self.enodeb_host.interfaces[0]
-        if "pdn_host" in main.params["UP4"]:
-            self.pdn_host = getattr(main, main.params["UP4"]["pdn_host"])
-            self.pdn_interface = self.pdn_host.interfaces[0]
-        self.router_mac = main.params["UP4"].get("router_mac", None)
+        if not no_host:
+            if "enodeb_host" in main.params["UP4"]:
+                self.enodeb_host = getattr(main,
+                                           main.params["UP4"]["enodeb_host"])
+                self.enodeb_interface = self.enodeb_host.interfaces[0]
+            if "pdn_host" in main.params["UP4"]:
+                self.pdn_host = getattr(main, main.params["UP4"]["pdn_host"])
+                self.pdn_interface = self.pdn_host.interfaces[0]
+            self.router_mac = main.params["UP4"].get("router_mac", None)
 
         # Start components
         self.up4_client.startP4RtClient()
@@ -201,6 +209,43 @@
         utilities.assert_equal(
             expect=False, actual=fail, onpass=msg, onfail=msg)
 
+    def readPdrsNumber(self):
+        """
+        Read the PDRs table and return the number of entries in the PDRs table
+
+        :return: Number of entries in the PDRs table
+        """
+        tableName = 'PreQosPipe.pdrs'
+        return self.up4_client.readNumberTableEntries(tableName)
+
+    def readFarsNumber(self):
+        """
+        Read the FARs table and return the number of entries in the FARs table
+
+        :return: Number of entries in the FARs table
+        """
+        tableName = 'PreQosPipe.load_far_attributes'
+        return self.up4_client.readNumberTableEntries(tableName)
+
+    def verifyUesFlowNumberP4rt(self):
+        """
+        Verify via P4RT CLI that the number of PDRs and FARs is the expected one
+
+        :return: True if the number of PDRs and FARs is expected, False otherwise
+        """
+        nPdrs = self.readPdrsNumber()
+        nFars = self.readFarsNumber()
+        return nPdrs == nFars == len(self.emulated_ues) * 2
+
+    def verifyNoUesFlowNumberP4rt(self, preInstalledUes=0):
+        """
+        Verify via P4RT CLI that there is no PDRs and FARs installed.
+
+        :param preInstalledUes: Number of UEs whose PDRs and FARs are still programmed
+        :return:
+        """
+        return self.readPdrsNumber() == self.readFarsNumber() == preInstalledUes * 2
+
     def verifyNoUesFlow(self, onosCli, retries=3):
         """
         Verify that no PDRs and FARs are installed in ONOS.