CORD-1583 Bug fixes for dual ToRs

Two things:
  - In dual (paired) ToR scenarios it is possible to have the same outport
    in multiple buckets in a hash group, as long as they have different labels.
    When adding buckets this was taken into account. But when removing buckets,
    only outport was being checked. This bug fix ensures that labels are checked
    as well when removing buckets.
  - In dual ToR scenarios, getting the right set of hash buckets proved difficult
    with existing 'retryHash' mechanism. Repealed and replaced with a bucket corrector
    mechanism that periodically corrects the hash group buckets when the topology
    has been stable for the last 10 secs. Required the introduction of a VERIFY
    operation in Next Objectives. Also added a cli command to trigger this
    operation manually.

Change-Id: Ib0d2734060fadc6e7a4bd0d75f3409e194413a97
diff --git a/core/api/src/main/java/org/onosproject/net/flowobjective/DefaultNextObjective.java b/core/api/src/main/java/org/onosproject/net/flowobjective/DefaultNextObjective.java
index c792525..671cc5f 100644
--- a/core/api/src/main/java/org/onosproject/net/flowobjective/DefaultNextObjective.java
+++ b/core/api/src/main/java/org/onosproject/net/flowobjective/DefaultNextObjective.java
@@ -247,25 +247,12 @@
 
         @Override
         public NextObjective add() {
-            treatments = listBuilder.build();
-            op = Operation.ADD;
-            checkNotNull(appId, "Must supply an application id");
-            checkNotNull(id, "id cannot be null");
-            checkNotNull(type, "The type cannot be null");
-            checkArgument(!treatments.isEmpty(), "Must have at least one treatment");
-
-            return new DefaultNextObjective(this);
+            return add(null);
         }
 
         @Override
         public NextObjective remove() {
-            treatments = listBuilder.build();
-            op = Operation.REMOVE;
-            checkNotNull(appId, "Must supply an application id");
-            checkNotNull(id, "id cannot be null");
-            checkNotNull(type, "The type cannot be null");
-
-            return new DefaultNextObjective(this);
+            return remove(null);
         }
 
         @Override
@@ -295,25 +282,12 @@
 
         @Override
         public NextObjective addToExisting() {
-            treatments = listBuilder.build();
-            op = Operation.ADD_TO_EXISTING;
-            checkNotNull(appId, "Must supply an application id");
-            checkNotNull(id, "id cannot be null");
-            checkNotNull(type, "The type cannot be null");
-            checkArgument(!treatments.isEmpty(), "Must have at least one treatment");
-
-            return new DefaultNextObjective(this);
+            return addToExisting(null);
         }
 
         @Override
         public NextObjective removeFromExisting() {
-            treatments = listBuilder.build();
-            op = Operation.REMOVE_FROM_EXISTING;
-            checkNotNull(appId, "Must supply an application id");
-            checkNotNull(id, "id cannot be null");
-            checkNotNull(type, "The type cannot be null");
-
-            return new DefaultNextObjective(this);
+            return removeFromExisting(null);
         }
 
         @Override
@@ -341,6 +315,22 @@
             return new DefaultNextObjective(this);
         }
 
+        @Override
+        public NextObjective verify() {
+            return verify(null);
+        }
+
+        @Override
+        public NextObjective verify(ObjectiveContext context) {
+            treatments = listBuilder.build();
+            op = Operation.VERIFY;
+            this.context = context;
+            checkNotNull(appId, "Must supply an application id");
+            checkNotNull(id, "id cannot be null");
+            checkNotNull(type, "The type cannot be null");
+            return new DefaultNextObjective(this);
+        }
+
     }
 
 }
diff --git a/core/api/src/main/java/org/onosproject/net/flowobjective/NextObjective.java b/core/api/src/main/java/org/onosproject/net/flowobjective/NextObjective.java
index 1c5ca95..c514baf 100644
--- a/core/api/src/main/java/org/onosproject/net/flowobjective/NextObjective.java
+++ b/core/api/src/main/java/org/onosproject/net/flowobjective/NextObjective.java
@@ -219,6 +219,22 @@
          */
         NextObjective removeFromExisting(ObjectiveContext context);
 
+        /**
+         * Builds the next objective that needs to be verified.
+         *
+         * @return a next objective with {@link Operation} VERIFY
+         */
+        NextObjective verify();
+
+        /**
+         * Builds the next objective that needs to be verified. The context will
+         * be used to notify the calling application.
+         *
+         * @param context an objective context
+         * @return a next objective with {@link Operation} VERIFY
+         */
+        NextObjective verify(ObjectiveContext context);
+
     }
 
 }
diff --git a/core/api/src/main/java/org/onosproject/net/flowobjective/Objective.java b/core/api/src/main/java/org/onosproject/net/flowobjective/Objective.java
index 20bf7b1..f0e4305 100644
--- a/core/api/src/main/java/org/onosproject/net/flowobjective/Objective.java
+++ b/core/api/src/main/java/org/onosproject/net/flowobjective/Objective.java
@@ -60,7 +60,20 @@
          * Remove from an existing Next Objective. Should not be used for any
          * other objective.
          */
-        REMOVE_FROM_EXISTING
+        REMOVE_FROM_EXISTING,
+
+        /**
+         * Verifies that an existing Next Objective's collection of treatments
+         * are correctly represented by the underlying implementation of the objective.
+         * Corrective action is taken if discrepancies are found during verification.
+         * For example, if the next objective defines 3 sets of treatments, which
+         * are meant to be implemented as 3 buckets in a group, but verification
+         * finds less or more buckets, then the appropriate buckets are added or
+         * removed to match the objective.
+         *
+         * Should not be used for any other objective.
+         */
+        VERIFY
     }
 
     /**