CORD-1180 Collection of fixes for hash-group buckets. Required the following changes:
	  Next-objectives that edited groups are now queued in the FlowObjectiveManager instead of the driver.
	  During linkup immediately checking for previous portups that should be added to a hash group.
	  A final retry 30 secs later to catch all ports that should be part of the same hash group.

Change-Id: I7ef450149d685890ca47932b8e559a0c11dc5ab4
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
index fce4e50..fec8b39 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
@@ -55,9 +55,13 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newScheduledThreadPool;
+import static org.onlab.util.Tools.groupedThreads;
 import static org.onosproject.segmentrouting.SegmentRoutingManager.INTERNAL_VLAN;
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -97,6 +101,10 @@
             portNextObjStore = null;
     private SegmentRoutingManager srManager;
 
+    private static final long RETRY_INTERVAL_SEC = 30;
+    private ScheduledExecutorService executorService
+    = newScheduledThreadPool(1, groupedThreads("retryhashbkts", "retry-%d", log));
+
     protected KryoNamespace.Builder kryo = new KryoNamespace.Builder()
             .register(URI.class).register(HashSet.class)
             .register(DeviceId.class).register(PortNumber.class)
@@ -183,7 +191,7 @@
      * discovered on this device.
      *
      * @param newLink new neighbor link
-     * @param isMaster true if local instance is the master
+     * @param isMaster true if local instance is the master for src-device of link
      *
      */
     public void linkUp(Link newLink, boolean isMaster) {
@@ -231,55 +239,79 @@
                 .filter((ns) -> (ns.getDeviceIds()
                         .contains(newLink.dst().deviceId())))
                 .collect(Collectors.toSet());
-        log.trace("linkUp: nsNextObjStore contents for device {}:",
-                deviceId,
-                nsSet);
+        log.debug("linkUp: nsNextObjStore contents for device {}: {}",
+                  deviceId, nsSet);
         for (NeighborSet ns : nsSet) {
             Integer nextId = nsNextObjStore.
                     get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
             if (nextId != null && isMaster) {
-                // Create the new bucket to be updated
-                TrafficTreatment.Builder tBuilder =
-                        DefaultTrafficTreatment.builder();
-                tBuilder.setOutput(newLink.src().port())
-                    .setEthDst(dstMac)
-                    .setEthSrc(nodeMacAddr);
-                if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
-                    tBuilder.pushMpls()
-                        .copyTtlOut()
-                        .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
+                addToHashedNextObjective(newLink.src().port(), dstMac, ns,
+                                         nextId, false);
+                // some links may have come up before the next-objective was created
+                // we take this opportunity to ensure other ports to same next-hop-dst
+                // are part of the hash group (see CORD-1180). Duplicate additions
+                // to the same hash group are avoided by the driver.
+                for (PortNumber p : devicePortMap.get(newLink.dst().deviceId())) {
+                    if (p.equals(newLink.src().port())) {
+                        continue;
+                    }
+                    addToHashedNextObjective(p, dstMac, ns, nextId, false);
                 }
-                // setup metadata to pass to nextObjective - indicate the vlan on egress
-                // if needed by the switch pipeline. Since hashed next-hops are always to
-                // other neighboring routers, there is no subnet assigned on those ports.
-                TrafficSelector.Builder metabuilder = DefaultTrafficSelector.builder();
-                metabuilder.matchVlanId(INTERNAL_VLAN);
-
-                NextObjective.Builder nextObjBuilder = DefaultNextObjective.builder()
-                        .withId(nextId)
-                        .withType(NextObjective.Type.HASHED)
-                        .addTreatment(tBuilder.build())
-                        .withMeta(metabuilder.build())
-                        .fromApp(appId);
-                log.info("**linkUp in device {}: Adding Bucket "
-                        + "with Port {} to next object id {}",
-                        deviceId,
-                        newLink.src().port(),
-                        nextId);
-
-                ObjectiveContext context = new DefaultObjectiveContext(
-                        (objective) -> log.debug("LinkUp addedTo NextObj {} on {}",
-                                nextId, deviceId),
-                        (objective, error) ->
-                                log.warn("LinkUp failed to addTo NextObj {} on {}: {}",
-                                        nextId, deviceId, error));
-                NextObjective nextObjective = nextObjBuilder.addToExisting(context);
-                flowObjectiveService.next(deviceId, nextObjective);
             } else if (isMaster) {
                 log.warn("linkUp in device {}, but global store has no record "
                         + "for neighbor-set {}", deviceId, ns);
             }
         }
+
+        // It's possible that at the time of linkup, some hash-groups have
+        // not been created yet by the instance responsible for creating them, or
+        // due to the eventually-consistent nature of the nsNextObjStore it has
+        // not synced up with this instance yet. Thus we perform this check again
+        // after a delay (see CORD-1180). Duplicate additions to the same hash group
+        // are avoided by the driver.
+        if (isMaster) {
+            executorService.schedule(new RetryHashBkts(newLink, dstMac),
+                                     RETRY_INTERVAL_SEC, TimeUnit.SECONDS);
+        }
+    }
+
+    private void addToHashedNextObjective(PortNumber outport, MacAddress dstMac,
+            NeighborSet ns, Integer nextId, boolean retry) {
+        // Create the new bucket to be updated
+        TrafficTreatment.Builder tBuilder =
+                DefaultTrafficTreatment.builder();
+        tBuilder.setOutput(outport)
+            .setEthDst(dstMac)
+            .setEthSrc(nodeMacAddr);
+        if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
+            tBuilder.pushMpls()
+                .copyTtlOut()
+                .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
+        }
+        // setup metadata to pass to nextObjective - indicate the vlan on egress
+        // if needed by the switch pipeline. Since hashed next-hops are always to
+        // other neighboring routers, there is no subnet assigned on those ports.
+        TrafficSelector.Builder metabuilder = DefaultTrafficSelector.builder();
+        metabuilder.matchVlanId(INTERNAL_VLAN);
+
+        NextObjective.Builder nextObjBuilder = DefaultNextObjective.builder()
+                .withId(nextId)
+                .withType(NextObjective.Type.HASHED)
+                .addTreatment(tBuilder.build())
+                .withMeta(metabuilder.build())
+                .fromApp(appId);
+        log.info("{} in device {}: Adding Bucket with Port {} to next object id {}",
+                 (retry) ? "**retry" : "**linkup",
+                         deviceId, outport, nextId);
+
+        ObjectiveContext context = new DefaultObjectiveContext(
+                (objective) -> log.debug("LinkUp addedTo NextObj {} on {}",
+                        nextId, deviceId),
+                (objective, error) ->
+                        log.warn("LinkUp failed to addTo NextObj {} on {}: {}",
+                                nextId, deviceId, error));
+        NextObjective nextObjective = nextObjBuilder.addToExisting(context);
+        flowObjectiveService.next(deviceId, nextObjective);
     }
 
     /**
@@ -735,7 +767,7 @@
                                     + " NextObj {} on {}: {}", nextId, deviceId, error)
                     );
             NextObjective nextObj = nextObjBuilder.add(context);
-            log.debug("**createGroupsFromNeighborsets: Submited "
+            log.debug("**createGroupsFromNeighborsets: Submitted "
                     + "next objective {} in device {}",
                     nextId, deviceId);
             flowObjectiveService.next(deviceId, nextObj);
@@ -907,4 +939,43 @@
         }
         // should probably clean local stores port-neighbor
     }
+
+    /**
+     * RetryHashBkts is a one-time retry at populating all the buckets of a
+     * hash group based on the given link. Should only be called by the
+     * master instance of the src-device of the link.
+     */
+    protected final class RetryHashBkts implements Runnable {
+        Link link;
+        MacAddress dstMac;
+
+        private RetryHashBkts(Link link, MacAddress dstMac) {
+            this.link = link;
+            this.dstMac = dstMac;
+        }
+
+        @Override
+        public void run() {
+            log.info("RETRY Hash buckets for linkup: {}", link);
+            Set<NeighborSet> nsSet = nsNextObjStore.keySet()
+                    .stream()
+                    .filter(nsStoreEntry -> nsStoreEntry.deviceId().equals(deviceId))
+                    .map(nsStoreEntry -> nsStoreEntry.neighborSet())
+                    .filter(ns -> ns.getDeviceIds()
+                            .contains(link.dst().deviceId()))
+                    .collect(Collectors.toSet());
+            log.debug("retry-link: nsNextObjStore contents for device {}: {}",
+                      deviceId, nsSet);
+            for (NeighborSet ns : nsSet) {
+                Integer nextId = nsNextObjStore.
+                        get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
+                if (nextId != null) {
+                    addToHashedNextObjective(link.src().port(), dstMac, ns,
+                                             nextId, true);
+                }
+            }
+        }
+    }
+
+
 }
diff --git a/cli/src/main/java/org/onosproject/cli/net/FlowObjectivePendingNextCommand.java b/cli/src/main/java/org/onosproject/cli/net/FlowObjectivePendingNextCommand.java
index 7350a6f..d1b5cc2 100644
--- a/cli/src/main/java/org/onosproject/cli/net/FlowObjectivePendingNextCommand.java
+++ b/cli/src/main/java/org/onosproject/cli/net/FlowObjectivePendingNextCommand.java
@@ -24,8 +24,8 @@
 
 /**
  * Returns a list of FlowObjective next-ids waiting to get created by device-drivers.
- * Also returns the forwarding objectives waiting on the pending next-objectives.
- * These lists are controller instance specific.
+ * Also returns the forwarding objectives and next objectives waiting on the pending
+ * next-objectives. These lists are controller instance specific.
  */
 @Command(scope = "onos", name = "obj-pending-nexts",
         description = "flow-objectives pending next-objectives")
@@ -37,7 +37,7 @@
     protected void execute() {
         try {
             FlowObjectiveService service = get(FlowObjectiveService.class);
-            printNexts(service.getPendingNexts());
+            printNexts(service.getPendingFlowObjectives());
         } catch (ServiceNotFoundException e) {
             print(FORMAT_MAPPING, "FlowObjectiveService unavailable");
         }
diff --git a/core/api/src/main/java/org/onosproject/net/flowobjective/FlowObjectiveService.java b/core/api/src/main/java/org/onosproject/net/flowobjective/FlowObjectiveService.java
index 722b881..e00e1b4 100644
--- a/core/api/src/main/java/org/onosproject/net/flowobjective/FlowObjectiveService.java
+++ b/core/api/src/main/java/org/onosproject/net/flowobjective/FlowObjectiveService.java
@@ -101,12 +101,24 @@
 
     /**
      * Retrieve all nextObjectives that are waiting to hear back from device
-     * drivers, and the forwarding-objectives that are waiting on the
-     * successful completion of the next-objectives. Consumed by the
+     * drivers, and the forwarding-objectives or next-objectives that are waiting
+     * on the successful completion of the original next-objectives. Consumed by the
      * "obj-pending-nexts" command on the CLI.
      *
      * @return a list of strings preformatted to provide information on the
      *          next-ids awaiting confirmation from the device-drivers.
      */
+    List<String> getPendingFlowObjectives();
+
+    /**
+     * Retrieve all nextObjectives that are waiting to hear back from device
+     * drivers, and the forwarding-objectives or next-objectives that are waiting
+     * on the successful completion of the original next-objectives.
+     *
+     * @return a list of strings preformatted by the device-drivers to provide
+     *         information on next-id to group-id mapping.
+     */
+    @Deprecated
     List<String> getPendingNexts();
+
 }
diff --git a/core/api/src/test/java/org/onosproject/net/flowobjective/FlowObjectiveServiceAdapter.java b/core/api/src/test/java/org/onosproject/net/flowobjective/FlowObjectiveServiceAdapter.java
index 9d5e806..fa43ab4 100644
--- a/core/api/src/test/java/org/onosproject/net/flowobjective/FlowObjectiveServiceAdapter.java
+++ b/core/api/src/test/java/org/onosproject/net/flowobjective/FlowObjectiveServiceAdapter.java
@@ -62,6 +62,11 @@
     }
 
     @Override
+    public List<String> getPendingFlowObjectives() {
+        return ImmutableList.of();
+    }
+
+    @Override
     public List<String> getPendingNexts() {
         return ImmutableList.of();
     }
diff --git a/core/net/src/main/java/org/onosproject/net/flowobjective/impl/FlowObjectiveManager.java b/core/net/src/main/java/org/onosproject/net/flowobjective/impl/FlowObjectiveManager.java
index 47b63ed..f51ee7c 100644
--- a/core/net/src/main/java/org/onosproject/net/flowobjective/impl/FlowObjectiveManager.java
+++ b/core/net/src/main/java/org/onosproject/net/flowobjective/impl/FlowObjectiveManager.java
@@ -48,6 +48,7 @@
 import org.onosproject.net.flowobjective.ForwardingObjective;
 import org.onosproject.net.flowobjective.NextObjective;
 import org.onosproject.net.flowobjective.Objective;
+import org.onosproject.net.flowobjective.Objective.Operation;
 import org.onosproject.net.flowobjective.ObjectiveError;
 import org.onosproject.net.flowobjective.ObjectiveEvent;
 import org.onosproject.net.flowobjective.ObjectiveEvent.Type;
@@ -126,7 +127,14 @@
 
     protected ServiceDirectory serviceDirectory = new DefaultServiceDirectory();
 
-    private final Map<Integer, Set<PendingNext>> pendingForwards = Maps.newConcurrentMap();
+    // local stores for queuing fwd and next objectives that are waiting for an
+    // associated next objective execution to complete. The signal for completed
+    // execution comes from a pipeline driver, in this or another controller
+    // instance, via the DistributedFlowObjectiveStore.
+    private final Map<Integer, Set<PendingFlowObjective>> pendingForwards =
+            Maps.newConcurrentMap();
+    private final Map<Integer, Set<PendingFlowObjective>> pendingNexts =
+            Maps.newConcurrentMap();
 
     // local store to track which nextObjectives were sent to which device
     // for debugging purposes
@@ -202,6 +210,7 @@
 
                 if (pipeliner != null) {
                     if (objective instanceof NextObjective) {
+                        nextToDevice.put(objective.id(), deviceId);
                         pipeliner.next((NextObjective) objective);
                     } else if (objective instanceof ForwardingObjective) {
                         pipeliner.forward((ForwardingObjective) objective);
@@ -218,7 +227,7 @@
                     objective.context().ifPresent(
                             c -> c.onError(objective, ObjectiveError.NOPIPELINER));
                 }
-                //Excpetion thrown
+                //Exception thrown
             } catch (Exception e) {
                 log.warn("Exception while installing flow objective", e);
             }
@@ -234,7 +243,7 @@
     @Override
     public void forward(DeviceId deviceId, ForwardingObjective forwardingObjective) {
         checkPermission(FLOWRULE_WRITE);
-        if (queueObjective(deviceId, forwardingObjective)) {
+        if (queueFwdObjective(deviceId, forwardingObjective)) {
             return;
         }
         executorService.execute(new ObjectiveInstaller(deviceId, forwardingObjective));
@@ -243,7 +252,9 @@
     @Override
     public void next(DeviceId deviceId, NextObjective nextObjective) {
         checkPermission(FLOWRULE_WRITE);
-        nextToDevice.put(nextObjective.id(), deviceId);
+        if (queueNextObjective(deviceId, nextObjective)) {
+            return;
+        }
         executorService.execute(new ObjectiveInstaller(deviceId, nextObjective));
     }
 
@@ -256,7 +267,7 @@
     @Override
     public void initPolicy(String policy) {}
 
-    private boolean queueObjective(DeviceId deviceId, ForwardingObjective fwd) {
+    private boolean queueFwdObjective(DeviceId deviceId, ForwardingObjective fwd) {
         if (fwd.nextId() == null ||
                 flowObjectiveStore.getNextGroup(fwd.nextId()) != null ||
                 fwd.op() == Objective.Operation.REMOVE) {
@@ -269,11 +280,11 @@
             // after a notification arrives
             if (flowObjectiveStore.getNextGroup(fwd.nextId()) == null) {
                 pendingForwards.compute(fwd.nextId(), (id, pending) -> {
-                    PendingNext next = new PendingNext(deviceId, fwd);
+                    PendingFlowObjective pendfo = new PendingFlowObjective(deviceId, fwd);
                     if (pending == null) {
-                        return Sets.newHashSet(next);
+                        return Sets.newHashSet(pendfo);
                     } else {
-                        pending.add(next);
+                        pending.add(pendfo);
                         return pending;
                     }
                 });
@@ -287,6 +298,38 @@
         return queued;
     }
 
+    private boolean queueNextObjective(DeviceId deviceId, NextObjective next) {
+        if (flowObjectiveStore.getNextGroup(next.id()) != null ||
+                next.op() == Operation.ADD) {
+            // either group exists or we are trying to create it - let it through
+            return false;
+        }
+        // we need to hold off on other operations till we get notified that the
+        // initial group creation has succeeded
+        boolean queued = false;
+        synchronized (pendingNexts) {
+            // double check the flow objective store, because this block could run
+            // after a notification arrives
+            if (flowObjectiveStore.getNextGroup(next.id()) == null) {
+                pendingNexts.compute(next.id(), (id, pending) -> {
+                    PendingFlowObjective pendfo = new PendingFlowObjective(deviceId, next);
+                    if (pending == null) {
+                        return Sets.newHashSet(pendfo);
+                    } else {
+                        pending.add(pendfo);
+                        return pending;
+                    }
+                });
+                queued = true;
+            }
+        }
+        if (queued) {
+            log.debug("Queued next objective {} with operation {} meant for device {}",
+                      next.id(), next.op(), deviceId);
+        }
+        return queued;
+    }
+
     /**
      * Retrieves (if it exists) the device pipeline behaviour from the cache.
      * Otherwise it warms the caches and triggers the init method of the Pipeline.
@@ -449,49 +492,70 @@
         public void notify(ObjectiveEvent event) {
             if (event.type() == Type.ADD) {
                 log.debug("Received notification of obj event {}", event);
-                Set<PendingNext> pending;
+                Set<PendingFlowObjective> pending;
+
+                // first send all pending flows
                 synchronized (pendingForwards) {
                     // needs to be synchronized for queueObjective lookup
                     pending = pendingForwards.remove(event.subject());
                 }
-
                 if (pending == null) {
-                    log.debug("Nothing pending for this obj event {}", event);
-                    return;
+                    log.debug("No forwarding objectives pending for this "
+                            + "obj event {}", event);
+                } else {
+                    log.debug("Processing {} pending forwarding objectives for nextId {}",
+                              pending.size(), event.subject());
+                    pending.forEach(p -> getDevicePipeliner(p.deviceId())
+                                    .forward((ForwardingObjective) p.flowObjective()));
                 }
 
-                log.debug("Processing {} pending forwarding objectives for nextId {}",
-                         pending.size(), event.subject());
-                pending.forEach(p -> getDevicePipeliner(p.deviceId())
-                                .forward(p.forwardingObjective()));
+                // now check for pending next-objectives
+                synchronized (pendingNexts) {
+                    // needs to be synchronized for queueObjective lookup
+                    pending = pendingNexts.remove(event.subject());
+                }
+                if (pending == null) {
+                    log.debug("No next objectives pending for this "
+                            + "obj event {}", event);
+                } else {
+                    log.debug("Processing {} pending next objectives for nextId {}",
+                              pending.size(), event.subject());
+                    pending.forEach(p -> getDevicePipeliner(p.deviceId())
+                                    .next((NextObjective) p.flowObjective()));
+                }
             }
         }
     }
 
     /**
-     * Data class used to hold a pending forwarding objective that could not
+     * Data class used to hold a pending flow objective that could not
      * be processed because the associated next object was not present.
+     * Note that this pending flow objective could be a forwarding objective
+     * waiting for a next objective to complete execution. Or it could a
+     * next objective (with a different operation - remove, addToExisting, or
+     * removeFromExisting) waiting for a next objective with the same id to
+     * complete execution.
      */
-    private class PendingNext {
+    private class PendingFlowObjective {
         private final DeviceId deviceId;
-        private final ForwardingObjective fwd;
+        private final Objective flowObj;
 
-        public PendingNext(DeviceId deviceId, ForwardingObjective fwd) {
+        public PendingFlowObjective(DeviceId deviceId, Objective flowObj) {
             this.deviceId = deviceId;
-            this.fwd = fwd;
+            this.flowObj = flowObj;
         }
 
         public DeviceId deviceId() {
             return deviceId;
         }
 
-        public ForwardingObjective forwardingObjective() {
-            return fwd;
+        public Objective flowObjective() {
+            return flowObj;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(deviceId, fwd);
+            return Objects.hash(deviceId, flowObj);
         }
 
         @Override
@@ -499,12 +563,12 @@
             if (this == obj) {
                 return true;
             }
-            if (!(obj instanceof PendingNext)) {
+            if (!(obj instanceof PendingFlowObjective)) {
                 return false;
             }
-            final PendingNext other = (PendingNext) obj;
+            final PendingFlowObjective other = (PendingFlowObjective) obj;
             if (this.deviceId.equals(other.deviceId) &&
-                    this.fwd.equals(other.fwd)) {
+                    this.flowObj.equals(other.flowObj)) {
                 return true;
             }
             return false;
@@ -541,24 +605,48 @@
     }
 
     @Override
-    public List<String> getPendingNexts() {
-        List<String> pendingNexts = new ArrayList<>();
-        for (Integer nextId : pendingForwards.keySet()) {
-            Set<PendingNext> pnext = pendingForwards.get(nextId);
+    public List<String> getPendingFlowObjectives() {
+        List<String> pendingFlowObjectives = new ArrayList<>();
 
+        for (Integer nextId : pendingForwards.keySet()) {
+            Set<PendingFlowObjective> pfwd = pendingForwards.get(nextId);
             StringBuilder pend = new StringBuilder();
             pend.append("NextId: ")
                     .append(nextId);
-            for (PendingNext pn : pnext) {
+            for (PendingFlowObjective pf : pfwd) {
                 pend.append("\n    FwdId: ")
-                        .append(String.format("%11s", pn.forwardingObjective().id()))
+                        .append(String.format("%11s", pf.flowObjective().id()))
+                        .append(", DeviceId: ")
+                        .append(pf.deviceId())
+                        .append(", Selector: ")
+                        .append(((ForwardingObjective) pf.flowObjective())
+                                    .selector().criteria());
+            }
+            pendingFlowObjectives.add(pend.toString());
+        }
+
+        for (Integer nextId : pendingNexts.keySet()) {
+            Set<PendingFlowObjective> pnext = pendingNexts.get(nextId);
+            StringBuilder pend = new StringBuilder();
+            pend.append("NextId: ")
+                    .append(nextId);
+            for (PendingFlowObjective pn : pnext) {
+                pend.append("\n    NextOp: ")
+                        .append(pn.flowObjective().op())
                         .append(", DeviceId: ")
                         .append(pn.deviceId())
-                        .append(", Selector: ")
-                        .append(pn.forwardingObjective().selector().criteria());
+                        .append(", Treatments: ")
+                        .append(((NextObjective) pn.flowObjective())
+                                    .next());
             }
-            pendingNexts.add(pend.toString());
+            pendingFlowObjectives.add(pend.toString());
         }
-        return pendingNexts;
+
+        return pendingFlowObjectives;
+    }
+
+    @Override
+    public List<String> getPendingNexts() {
+        return getPendingFlowObjectives();
     }
 }
diff --git a/core/net/src/main/java/org/onosproject/net/flowobjective/impl/composition/FlowObjectiveCompositionManager.java b/core/net/src/main/java/org/onosproject/net/flowobjective/impl/composition/FlowObjectiveCompositionManager.java
index 3e0d2eb..60da0e6 100644
--- a/core/net/src/main/java/org/onosproject/net/flowobjective/impl/composition/FlowObjectiveCompositionManager.java
+++ b/core/net/src/main/java/org/onosproject/net/flowobjective/impl/composition/FlowObjectiveCompositionManager.java
@@ -432,8 +432,13 @@
     }
 
     @Override
-    public List<String> getPendingNexts() {
+    public List<String> getPendingFlowObjectives() {
         // TODO Implementation deferred as this is an experimental component.
         return ImmutableList.of();
     }
+
+    @Override
+    public List<String> getPendingNexts() {
+        return ImmutableList.of();
+    }
 }
diff --git a/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2GroupHandler.java b/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2GroupHandler.java
index aaca495..6a03ee6 100644
--- a/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2GroupHandler.java
+++ b/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2GroupHandler.java
@@ -984,7 +984,8 @@
      * @param nextObjective the objective to process.
      */
     protected void processPwNextObjective(NextObjective nextObjective) {
-        log.warn("Pseudo wire extensions are not support for the OFDPA 2.0 {}", nextObjective.id());
+        log.warn("Pseudo wire extensions are not supported in OFDPA 2.0 {}",
+                 nextObjective.id());
     }
 
     //////////////////////////////////////
@@ -1049,6 +1050,9 @@
             objectiveToAdd = builder.addToExisting(context);
         } else {
             // buckets to add are already there - nothing to do
+            log.debug("buckets already exist {} in next: {} ..ignoring bucket add",
+                      duplicateBuckets, nextObjective.id());
+            pass(nextObjective);
             return;
         }
 
diff --git a/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2Pipeline.java b/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2Pipeline.java
index 1abe86c..9800260 100644
--- a/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2Pipeline.java
+++ b/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2Pipeline.java
@@ -283,13 +283,13 @@
                          nextObjective.id(), deviceId);
                 return;
             }
-            log.debug("Processing NextObjective id{} in dev{} - add group",
+            log.debug("Processing NextObjective id {} in dev {} - add group",
                       nextObjective.id(), deviceId);
             groupHandler.addGroup(nextObjective);
             break;
         case ADD_TO_EXISTING:
             if (nextGroup != null) {
-                log.debug("Processing NextObjective id{} in dev{} - add bucket",
+                log.debug("Processing NextObjective id {} in dev {} - add bucket",
                           nextObjective.id(), deviceId);
                 groupHandler.addBucketToGroup(nextObjective, nextGroup);
             } else {
@@ -313,7 +313,7 @@
                          nextObjective.id(), deviceId);
                 return;
             }
-            log.debug("Processing NextObjective id{}  in dev{} - remove group",
+            log.debug("Processing NextObjective id {}  in dev {} - remove group",
                       nextObjective.id(), deviceId);
             groupHandler.removeGroup(nextObjective, nextGroup);
             break;
@@ -323,7 +323,7 @@
                          nextObjective.id(), deviceId);
                 return;
             }
-            log.debug("Processing NextObjective id{} in dev{} - remove bucket",
+            log.debug("Processing NextObjective id {} in dev {} - remove bucket",
                       nextObjective.id(), deviceId);
             groupHandler.removeBucketFromGroup(nextObjective, nextGroup);
             break;
diff --git a/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/OvsOfdpa2Pipeline.java b/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/OvsOfdpa2Pipeline.java
index b0e3d9f..4a6d7d9 100644
--- a/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/OvsOfdpa2Pipeline.java
+++ b/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/OvsOfdpa2Pipeline.java
@@ -20,11 +20,7 @@
 
 /**
  * Driver for software switch emulation of the OFDPA pipeline.
- * The software switch is the OVS OF 1.3 switch. Unfortunately the OVS switch
- * does not handle vlan tags and mpls labels simultaneously, which requires us
- * to do some workarounds in the driver. This driver is meant for the use of
- * the cpqd switch when MPLS is required. As a result this driver works only
- * on incoming untagged packets.
+ * The software switch is the OVS OF 1.3 switch (version 2.5 or later).
  */
 public class OvsOfdpa2Pipeline extends CpqdOfdpa2Pipeline {
 
diff --git a/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowObjectiveManager.java b/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowObjectiveManager.java
index 8eba5cf..b28a34e 100644
--- a/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowObjectiveManager.java
+++ b/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowObjectiveManager.java
@@ -185,7 +185,7 @@
     }
 
     @Override
-    public List<String> getPendingNexts() {
+    public List<String> getPendingFlowObjectives() {
         List<String> pendingNexts = new ArrayList<>();
         for (Integer nextId : pendingForwards.keySet()) {
             Set<PendingNext> pnext = pendingForwards.get(nextId);
@@ -201,6 +201,11 @@
         return pendingNexts;
     }
 
+    @Override
+    public List<String> getPendingNexts() {
+        return getPendingFlowObjectives();
+    }
+
     private boolean queueObjective(DeviceId deviceId, ForwardingObjective fwd) {
         if (fwd.nextId() == null ||
                 flowObjectiveStore.getNextGroup(fwd.nextId()) != null) {
@@ -336,10 +341,12 @@
 
     // Processing context for initializing pipeline driver behaviours.
     private class InnerPipelineContext implements PipelinerContext {
+        @Override
         public ServiceDirectory directory() {
             return serviceDirectory;
         }
 
+        @Override
         public FlowObjectiveStore store() {
             return flowObjectiveStore;
         }
@@ -625,4 +632,5 @@
             }
         }
     }
+
 }