Fix dump flows to ease debugging in UP4 and QoS tests

Change-Id: I84fc7f003c674eb5ad8c1c365741eaf69f155f3c
diff --git a/TestON/tests/USECASE/SegmentRouting/QOS/QOS.py b/TestON/tests/USECASE/SegmentRouting/QOS/QOS.py
index 315328f..1b20ce8 100644
--- a/TestON/tests/USECASE/SegmentRouting/QOS/QOS.py
+++ b/TestON/tests/USECASE/SegmentRouting/QOS/QOS.py
@@ -28,6 +28,7 @@
         main.step("Start P4rt client and setup TRex")
         # Use the first available ONOS instance CLI
         onos_cli = main.Cluster.active(0).CLI
+        initial_flow_count = onos_cli.checkFlowCount()
         up4 = UP4()
         trex = Trex()
         # Get the P4RT client connected to UP4 in the first available ONOS instance
@@ -40,6 +41,11 @@
         main.step("Verify PDRs and FARs in ONOS")
         up4.verifyUp4Flow(onos_cli)
 
+        run.checkFlows(
+            main,
+            minFlowCount=initial_flow_count+(len(up4.emulated_ues)*2)
+        )
+
         # Load traffic config for the current test case
         main.step("Load test JSON config")
         cfgFile = main.configPath + "/tests/" + "leaf_edge_mobile.json"
@@ -72,6 +78,8 @@
         main.step("Verify removed PDRs and FARs from ONOS")
         up4.verifyNoUesFlow(onos_cli)
 
+        run.checkFlows(main, minFlowCount=initial_flow_count)
+
         main.step("Teardown")
         trex.teardown()
         up4.teardown()
diff --git a/TestON/tests/USECASE/SegmentRouting/UP4/UP4.py b/TestON/tests/USECASE/SegmentRouting/UP4/UP4.py
index 556304b..1395671 100644
--- a/TestON/tests/USECASE/SegmentRouting/UP4/UP4.py
+++ b/TestON/tests/USECASE/SegmentRouting/UP4/UP4.py
@@ -32,6 +32,7 @@
         main.step("Start scapy and p4rt client")
         # Use the first available ONOS instance CLI
         onos_cli = main.Cluster.active(0).CLI
+        initial_flow_count = onos_cli.checkFlowCount()
         up4 = UP4()
         # Get the P4RT client connected to UP4 in the first available ONOS instance
         up4.setup(main.Cluster.active(0).p4rtUp4)
@@ -40,6 +41,11 @@
         up4.attachUes()
         up4.verifyUp4Flow(onos_cli)
 
+        run.checkFlows(
+            main,
+            minFlowCount=initial_flow_count+(len(up4.emulated_ues)*2)
+        )
+
         # ------- Test Upstream traffic (enb->pdn)
         main.step("Test upstream traffic")
         up4.testUpstreamTraffic()
@@ -52,11 +58,13 @@
         up4.detachUes()
         up4.verifyNoUesFlow(onos_cli)
 
+        run.checkFlows(main, minFlowCount=initial_flow_count)
+
         main.step("Stop scapy and p4rt client")
         up4.teardown()
         run.cleanup(main)
 
-    def CASE2(self):
+    def CASE2(self, main):
         main.case("BESS traffic routed")
         """
         Program PDRs and FARs for UEs managed via UP4
@@ -90,6 +98,7 @@
         main.step("Start scapy and p4rt client + Scapy on BESS Host")
         # Use the first available ONOS instance CLI
         onos_cli = main.Cluster.active(0).CLI
+        initial_flow_count = onos_cli.checkFlowCount()
         up4 = UP4()
         # Get the P4RT client connected to UP4 in the first available ONOS instance
         up4.setup(main.Cluster.active(0).p4rtUp4)
@@ -108,6 +117,11 @@
         up4.attachUes()
         up4.verifyUp4Flow(onos_cli)
 
+        run.checkFlows(
+            main,
+            minFlowCount=initial_flow_count+(len(up4.emulated_ues)*2)
+        )
+
         # ------------------- UPSTREAM -------------------
         # ------- eNB -> fabric -> BESS (encapped)
         main.step("Test upstream eNB -> fabric -> BESS")
@@ -237,6 +251,8 @@
         up4.detachUes()
         up4.verifyNoUesFlow(onos_cli)
 
+        run.checkFlows(main, minFlowCount=initial_flow_count)
+
         main.step("Stop scapy and p4rt client")
         up4.teardown()
         bess_host.stopScapy()
@@ -274,6 +290,7 @@
         up4_0 = UP4()
         up4_1 = UP4()
         up4_2 = UP4()
+        initial_flow_count = onos_cli_0.checkFlowCount()
 
         main.step("Program and Verify PDRs and FARs via UP4 on ONOS 0")
         up4_0.setup(main.Cluster.active(0).p4rtUp4, no_host=True)
@@ -281,6 +298,11 @@
         up4_0.verifyUp4Flow(onos_cli_0)
         up4_0.teardown()
 
+        run.checkFlows(
+            main,
+            minFlowCount=initial_flow_count+(len(up4_0.emulated_ues)*2)
+        )
+
         main.step("Verify PDRs and FARs number via UP4 P4RT on ONOS 1")
         up4_1.setup(main.Cluster.active(1).p4rtUp4, no_host=True)
         utilities.assert_equal(
@@ -316,6 +338,8 @@
         up4_2.detachUes()
         up4_2.verifyNoUesFlow(onos_cli_2)
 
+        run.checkFlows(main, minFlowCount=initial_flow_count)
+
         main.step("Verify no PDRs and FARs via UP4 P4RT on ONOS 2")
         utilities.assert_equal(
             expect=True,
@@ -394,12 +418,19 @@
         up4_1 = UP4()
         up4_2 = UP4()
 
+        initial_flow_count = onos_cli_0.checkFlowCount()
+
         main.step("Program and Verify PDRs and FARs via UP4 on ONOS 0")
         up4_0.setup(main.Cluster.active(0).p4rtUp4, no_host=True)
         up4_0.attachUes()
         up4_0.verifyUp4Flow(onos_cli_0)
         up4_0.teardown()
 
+        run.checkFlows(
+            main,
+            minFlowCount=initial_flow_count+(len(up4_0.emulated_ues)*2)
+        )
+
         onosPod = main.params["UP4_delete_pod"]
         # Exit from previous port forwarding, because we need to restore
         # port-forwarding after ONOS reboot.
@@ -439,6 +470,17 @@
         )
         up4_2.teardown()
 
+        main.step("Verify all active ONOS instances have the same number of flows")
+        onos_1_flow_count = onos_cli_1.checkFlowCount()
+        onos_2_flow_count = onos_cli_2.checkFlowCount()
+        utilities.assert_equal(
+            expect=True,
+            actual=onos_1_flow_count == onos_2_flow_count,
+            onpass="All ONOS instances have the same number of flows",
+            onfail="ONOS instances have different number of flows: (%d, %d)" % (
+                onos_1_flow_count, onos_2_flow_count)
+        )
+
         main.step("Verify ONOS 0 has restarted correctly")
         onosStarted = utilities.retry(
             f=kubectl_0.kubectlCheckPodReady,
@@ -498,11 +540,18 @@
             onfail="Stale PDRs and FARs"
         )
 
+        run.checkFlows(main, minFlowCount=initial_flow_count)
+
         main.step("Re-program PDRs and FARs via UP4 on ONOS 0 after restart")
         up4_0.attachUes()
         up4_0.verifyUp4Flow(onos_cli_0)
         up4_0.teardown()
 
+        run.checkFlows(
+            main,
+            minFlowCount=initial_flow_count+(len(up4_0.emulated_ues)*2)
+        )
+
         main.step("Verify PDRs and FARs via UP4 on ONOS 1")
         up4_1.setup(main.Cluster.active(1).p4rtUp4, no_host=True)
         up4_1.verifyUp4Flow(onos_cli_1)
@@ -524,4 +573,6 @@
         up4_1.verifyNoUesFlow(onos_cli_1)
         up4_1.teardown()
 
+        run.checkFlows(main, minFlowCount=initial_flow_count)
+
         run.cleanup(main)