CORD-1419 CORD-1425 CORD-1496 CORD-639 Changes for dual-ToRs

Introduces the concept of edge-pairs (or paired-ToRs) which
can have some subnets/prefixes reachable by both ToRs.
   - Each ToR can also have prefixes reachable only by itself,
     even though it is part of an edge-pair.
   - The paired link between ToRs in an edge-pair is ignored
     for ECMP calculations.
   - Required a change in how destinations and next-hops are stored.
     The neighborSet is now a destinationSet, and no longer carries
     next-hop info, which is now stored in NextNeighbors. As a result,
     the DestinationSetNextObjectiveStoreKey and ECMP group id do not
     change as next-hops come and go.
   - It is now possible to have buckets in hash groups with the same
     outport but different labels.
   - DefaultRoutingHandler has been rearraged to be more readable, and
     clearly highlight the three major ways that routing changes can
     happen in the network.

Also fixes the case where config is added after switches connect to the controller.

Change-Id: I7ce93ab201f6ef2c01cbe07a51ee78cd6a0a112e
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
index bfb1a8b..85b0c76 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
@@ -16,6 +16,7 @@
 package org.onosproject.segmentrouting.grouphandler;
 
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 
@@ -42,14 +43,13 @@
 import org.onosproject.segmentrouting.SegmentRoutingManager;
 import org.onosproject.segmentrouting.config.DeviceConfigNotFoundException;
 import org.onosproject.segmentrouting.config.DeviceProperties;
-import org.onosproject.segmentrouting.storekey.NeighborSetNextObjectiveStoreKey;
+import org.onosproject.segmentrouting.storekey.DestinationSetNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.PortNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.VlanNextObjectiveStoreKey;
 import org.onosproject.store.service.EventuallyConsistentMap;
 import org.slf4j.Logger;
 
 import java.net.URI;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -98,9 +98,9 @@
     protected ConcurrentHashMap<PortNumber, DeviceId> portDeviceMap =
             new ConcurrentHashMap<>();
 
-    // distributed store for (device+neighborset) mapped to next-id
-    protected EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey, Integer>
-            nsNextObjStore = null;
+    // distributed store for (device+destination-set) mapped to next-id and neighbors
+    protected EventuallyConsistentMap<DestinationSetNextObjectiveStoreKey, NextNeighbors>
+            dsNextObjStore = null;
     // distributed store for (device+subnet-ip-prefix) mapped to next-id
     protected EventuallyConsistentMap<VlanNextObjectiveStoreKey, Integer>
             vlanNextObjStore = null;
@@ -115,8 +115,8 @@
 
     protected KryoNamespace.Builder kryo = new KryoNamespace.Builder()
             .register(URI.class).register(HashSet.class)
-            .register(PortNumber.class)
-            .register(NeighborSet.class).register(PolicyGroupIdentifier.class)
+            .register(DeviceId.class).register(PortNumber.class)
+            .register(DestinationSet.class).register(PolicyGroupIdentifier.class)
             .register(PolicyGroupParams.class)
             .register(GroupBucketIdentifier.class)
             .register(GroupBucketIdentifier.BucketOutputType.class);
@@ -141,7 +141,7 @@
                     + " Skipping value assignment in DefaultGroupHandler");
         }
         this.flowObjectiveService = flowObjService;
-        this.nsNextObjStore = srManager.nsNextObjStore();
+        this.dsNextObjStore = srManager.dsNextObjStore();
         this.vlanNextObjStore = srManager.vlanNextObjStore();
         this.portNextObjStore = srManager.portNextObjStore();
         this.srManager = srManager;
@@ -231,28 +231,33 @@
             log.warn(e.getMessage() + " Aborting retryHash.");
             return;
         }
-        // find all the neighborSets related to link
-        Set<NeighborSet> nsSet = nsNextObjStore.keySet()
+        // find all the destinationSets related to link
+        Set<DestinationSetNextObjectiveStoreKey> dsKeySet = dsNextObjStore.entrySet()
                 .stream()
-                .filter((nsStoreEntry) -> (nsStoreEntry.deviceId().equals(deviceId)))
-                .map((nsStoreEntry) -> (nsStoreEntry.neighborSet()))
-                .filter((ns) -> (ns.getDeviceIds()
-                        .contains(link.dst().deviceId())))
+                .filter(entry -> entry.getKey().deviceId().equals(deviceId))
+                .filter(entry -> entry.getValue().containsNextHop(link.dst().deviceId()))
+                .map(entry -> entry.getKey())
                 .collect(Collectors.toSet());
-        log.debug("retryHash: nsNextObjStore contents for linkSrc {} -> linkDst {}: {}",
-                  deviceId, link.dst().deviceId(), nsSet);
 
-        for (NeighborSet ns : nsSet) {
-            Integer nextId = nsNextObjStore.
-                    get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-            if (nextId == null) {
+        log.debug("retryHash: dsNextObjStore contents for linkSrc {} -> linkDst {}: {}",
+                  deviceId, link.dst().deviceId(), dsKeySet);
+
+        for (DestinationSetNextObjectiveStoreKey dsKey : dsKeySet) {
+            NextNeighbors nextHops = dsNextObjStore.get(dsKey);
+            if (nextHops == null) {
                 log.warn("retryHash in device {}, but global store has no record "
-                        + "for neighbor-set {}", deviceId, ns);
+                         + "for dsKey:{}", deviceId, dsKey);
                 continue;
             }
+            int nextId = nextHops.nextId();
+            Set<DeviceId> dstSet = nextHops.getDstForNextHop(link.dst().deviceId());
             if (!linkDown) {
-                addToHashedNextObjective(link.src().port(), dstMac, ns,
-                                         nextId, false);
+                dstSet.forEach(dst -> {
+                    int edgeLabel = dsKey.destinationSet().getEdgeLabel(dst);
+                    addToHashedNextObjective(link.src().port(), dstMac,
+                                             edgeLabel, nextId, false);
+                });
+
                 if (firstTime) {
                     // some links may have come up before the next-objective was created
                     // we take this opportunity to ensure other ports to same next-hop-dst
@@ -262,12 +267,19 @@
                         if (p.equals(link.src().port())) {
                             continue;
                         }
-                        addToHashedNextObjective(p, dstMac, ns, nextId, false);
+                        dstSet.forEach(dst -> {
+                            int edgeLabel = dsKey.destinationSet().getEdgeLabel(dst);
+                            addToHashedNextObjective(p, dstMac, edgeLabel,
+                                                     nextId, false);
+                        });
                     }
                 }
             } else {
-                removeFromHashedNextObjective(link.src().port(), dstMac, ns,
-                                              nextId);
+                dstSet.forEach(dst -> {
+                    int edgeLabel = dsKey.destinationSet().getEdgeLabel(dst);
+                    removeFromHashedNextObjective(link.src().port(), dstMac,
+                                                  edgeLabel, nextId);
+                });
             }
         }
 
@@ -289,23 +301,23 @@
      *
      * @param outport port to add to hash group
      * @param dstMac destination mac address of next-hop
-     * @param ns neighbor set representing next-hops and destination switch
+     * @param edgeLabel the label to use in the bucket
      * @param nextId id for next-objective to which the bucket will be added
      * @param retry indicates if this method is being called on a retry attempt
      *              at adding a bucket to the group
      */
     private void addToHashedNextObjective(PortNumber outport, MacAddress dstMac,
-            NeighborSet ns, Integer nextId, boolean retry) {
+            int edgeLabel, Integer nextId, boolean retry) {
         // Create the new bucket to be updated
         TrafficTreatment.Builder tBuilder =
                 DefaultTrafficTreatment.builder();
         tBuilder.setOutput(outport)
             .setEthDst(dstMac)
             .setEthSrc(nodeMacAddr);
-        if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
+        if (edgeLabel != DestinationSet.NO_EDGE_LABEL) {
             tBuilder.pushMpls()
                 .copyTtlOut()
-                .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
+                .setMpls(MplsLabel.mplsLabel(edgeLabel));
         }
         // setup metadata to pass to nextObjective - indicate the vlan on egress
         // if needed by the switch pipeline. Since hashed next-hops are always to
@@ -319,9 +331,9 @@
                 .addTreatment(tBuilder.build())
                 .withMeta(metabuilder.build())
                 .fromApp(appId);
-        log.debug("{} in device {}: Adding Bucket with Port {} to next object id {}",
+        log.debug("{} in device {}: Adding Bucket with port/label {}/{} to nextId {}",
                  (retry) ? "retry-addToHash" : "addToHash",
-                         deviceId, outport, nextId);
+                         deviceId, outport, edgeLabel, nextId);
 
         ObjectiveContext context = new DefaultObjectiveContext(
                 (objective) -> log.debug("{} addedTo NextObj {} on {}",
@@ -341,21 +353,21 @@
     *
     * @param port port to remove from hash group
     * @param dstMac destination mac address of next-hop
-    * @param ns neighbor set representing next-hops and destination switch
+    * @param edgeLabel the label to use in the bucket
     * @param nextId id for next-objective from which the bucket will be removed
     */
    private void removeFromHashedNextObjective(PortNumber port, MacAddress dstMac,
-                                              NeighborSet ns, Integer nextId) {
+                                              int edgeLabel, Integer nextId) {
        // Create the bucket to be removed
        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
                .builder();
        tBuilder.setOutput(port)
            .setEthDst(dstMac)
            .setEthSrc(nodeMacAddr);
-       if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
+       if (edgeLabel != DestinationSet.NO_EDGE_LABEL) {
            tBuilder.pushMpls()
                .copyTtlOut()
-               .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
+               .setMpls(MplsLabel.mplsLabel(edgeLabel));
        }
        log.info("{} in device {}: Removing Bucket with Port {} to next object id {}",
                 "removeFromHash", deviceId, port, nextId);
@@ -395,20 +407,22 @@
     public boolean fixHashGroups(DeviceId targetSw, Set<DeviceId> nextHops,
                                  DeviceId destSw, boolean revoke) {
         // temporary storage of keys to be updated
-        Map<NeighborSetNextObjectiveStoreKey, Set<DeviceId>> tempStore =
+        Map<DestinationSetNextObjectiveStoreKey, Set<DeviceId>> tempStore =
                 new HashMap<>();
         boolean foundNextObjective = false;
 
-        // retrieve hash-groups meant for destSw, which have neighborSets
+        // retrieve hash-groups meant for destSw, which have destinationSets
         // with different neighbors than the given next-hops
-        for (NeighborSetNextObjectiveStoreKey nskey : nsNextObjStore.keySet()) {
-            if (!nskey.deviceId().equals(targetSw) ||
-                    !nskey.neighborSet().getDestinationSw().equals(destSw)) {
+        for (DestinationSetNextObjectiveStoreKey dskey : dsNextObjStore.keySet()) {
+            if (!dskey.deviceId().equals(targetSw) ||
+                    !dskey.destinationSet().getDestinationSwitches().contains(destSw)) {
                 continue;
             }
             foundNextObjective = true;
-            Set<DeviceId> currNeighbors = nskey.neighborSet().getDeviceIds();
-            Integer nextId = nsNextObjStore.get(nskey);
+            NextNeighbors nhops = dsNextObjStore.get(dskey);
+            Set<DeviceId> currNeighbors = nhops.nextHops(destSw);
+            int edgeLabel = dskey.destinationSet().getEdgeLabel(destSw);
+            Integer nextId = nhops.nextId();
 
             Set<DeviceId> diff;
             if (revoke) {
@@ -428,7 +442,7 @@
                     dstMac = deviceConfig.getDeviceMac(neighbor);
                 } catch (DeviceConfigNotFoundException e) {
                     log.warn(e.getMessage() + " Aborting fixHashGroup for nextId:"
-                            + nskey);
+                            + nextId);
                     return false;
                 }
                 if (devicePortMap.get(neighbor) == null ||
@@ -444,22 +458,22 @@
                                 + "with Port {} to next object id {}",
                                 deviceId, port, nextId);
                         removeFromHashedNextObjective(port, dstMac,
-                                                      nskey.neighborSet(),
+                                                      edgeLabel,
                                                       nextId);
                     }
                     // to update neighbor set with changes made
-                    tempStore.put(nskey, Sets.difference(currNeighbors, diff));
+                    tempStore.put(dskey, Sets.difference(currNeighbors, diff));
                 } else {
                     for (PortNumber port : devicePortMap.get(neighbor)) {
                         log.info("fixHashGroup in device {}: Adding Bucket "
                                 + "with Port {} to next object id {}",
                                 deviceId, port, nextId);
                         addToHashedNextObjective(port, dstMac,
-                                                 nskey.neighborSet(),
+                                                 edgeLabel,
                                                  nextId, false);
                     }
                     // to update neighbor set with changes made
-                    tempStore.put(nskey, Sets.union(currNeighbors, diff));
+                    tempStore.put(dskey, Sets.union(currNeighbors, diff));
                 }
             }
         }
@@ -470,30 +484,102 @@
             return true; // nothing to do, return true so ECMPspg is updated
         }
 
-        // update the nsNextObjectiveStore with new neighborSets to nextId mappings
-        for (NeighborSetNextObjectiveStoreKey oldkey : tempStore.keySet()) {
-            Integer nextId = nsNextObjStore.get(oldkey);
-            if (nextId == null) {
+        // update the dsNextObjectiveStore with new destinationSet to nextId mappings
+        for (DestinationSetNextObjectiveStoreKey key : tempStore.keySet()) {
+            NextNeighbors oldHops = dsNextObjStore.get(key);
+            if (oldHops == null) {
                 continue;
             }
-            Set<DeviceId> newNeighbors = tempStore.get(oldkey);
-            NeighborSet newNs = new NeighborSet(newNeighbors,
-                                                oldkey.neighborSet().mplsSet(),
-                                                oldkey.neighborSet().getEdgeLabel(),
-                                                oldkey.neighborSet().getDestinationSw());
-            NeighborSetNextObjectiveStoreKey newkey =
-                    new NeighborSetNextObjectiveStoreKey(deviceId, newNs);
-            log.debug("Updating nsNextObjStore: oldKey:{} -> newKey:{} :: nextId:{}",
-                      oldkey, newkey, nextId);
-            synchronized (nsNextObjStore) {
-                nsNextObjStore.remove(oldkey);
-                nsNextObjStore.put(newkey, nextId);
-            }
+            Set<DeviceId> newNeighbors = tempStore.get(key);
+            Set<DeviceId> oldNeighbors = ImmutableSet.copyOf(oldHops.nextHops(destSw));
+            oldHops.dstNextHops().put(destSw, newNeighbors);
+            log.debug("Updating nsNextObjStore: oldHops:{} -> newHops:{} :: nextId:{}",
+                      oldNeighbors, newNeighbors, oldHops.nextId());
         }
 
         return true;
     }
 
+
+    public boolean updateNextHops(DestinationSet ds,
+                                  Map<DeviceId, Set<DeviceId>> newDstNextHops) {
+        DestinationSetNextObjectiveStoreKey key =
+                new DestinationSetNextObjectiveStoreKey(deviceId, ds);
+        NextNeighbors currNext = dsNextObjStore.get(key);
+        Map<DeviceId, Set<DeviceId>> currDstNextHops = currNext.dstNextHops();
+
+        // add newDstNextHops to currDstNextHops for each dst
+        boolean success = true;
+        for (DeviceId dstSw : ds.getDestinationSwitches()) {
+            Set<DeviceId> currNhops = currDstNextHops.get(dstSw);
+            Set<DeviceId> newNhops = newDstNextHops.get(dstSw);
+            currNhops = (currNhops == null) ? Sets.newHashSet() : currNhops;
+            newNhops = (newNhops == null) ? Sets.newHashSet() : newNhops;
+            int edgeLabel = ds.getEdgeLabel(dstSw);
+            int nextId = currNext.nextId();
+
+            // new next hops should be added
+            boolean suc = updateAllPortsToNextHop(Sets.difference(newNhops, currNhops),
+                                                  edgeLabel, nextId, false);
+            if (suc) {
+                currNhops.addAll(newNhops);
+                currDstNextHops.put(dstSw, currNhops); // this is only a local change
+            }
+            success &= suc;
+        }
+
+        if (success) {
+            // update global store
+            dsNextObjStore.put(key, new NextNeighbors(currDstNextHops,
+                                                      currNext.nextId()));
+            log.debug("Updated device:{} ds:{} new next-hops: {}", deviceId, ds,
+                      dsNextObjStore.get(key));
+        }
+        return success;
+    }
+
+    private boolean updateAllPortsToNextHop(Set<DeviceId> diff, int edgeLabel,
+                                         int nextId, boolean revoke) {
+        for (DeviceId neighbor : diff) {
+            MacAddress dstMac;
+            try {
+                dstMac = deviceConfig.getDeviceMac(neighbor);
+            } catch (DeviceConfigNotFoundException e) {
+                log.warn(e.getMessage() + " Aborting fixHashGroup for nextId:"
+                        + nextId);
+                return false;
+            }
+            if (devicePortMap.get(neighbor) == null ||
+                    devicePortMap.get(neighbor).isEmpty()) {
+                log.warn("No ports found in dev:{} for neighbor:{} .. cannot "
+                        + "fix hash group for nextId: {}",
+                         deviceId, neighbor, nextId);
+                return false;
+            }
+            if (revoke) {
+                for (PortNumber port : devicePortMap.get(neighbor)) {
+                    log.debug("fixHashGroup in device {}: Removing Bucket "
+                            + "with Port {} edgeLabel:{} to next object id {}",
+                            deviceId, port, edgeLabel, nextId);
+                    removeFromHashedNextObjective(port, dstMac,
+                                                  edgeLabel,
+                                                  nextId);
+                }
+            } else {
+                for (PortNumber port : devicePortMap.get(neighbor)) {
+                    log.debug("fixHashGroup in device {}: Adding Bucket "
+                            + "with Port {} edgeLabel: {} to next object id {}",
+                            deviceId, port, edgeLabel, nextId);
+                    addToHashedNextObjective(port, dstMac,
+                                             edgeLabel,
+                                             nextId, false);
+                }
+            }
+        }
+        return true;
+    }
+
+
     /**
      * Adds or removes a port that has been configured with a vlan to a broadcast group
      * for bridging. Should only be called by the master instance for this device.
@@ -558,45 +644,61 @@
     }
 
     /**
-     * Returns the next objective of type hashed associated with the neighborset.
-     * If there is no next objective for this neighborset, this method
-     * would create a next objective and return. Optionally metadata can be
+     * Returns the next objective of type hashed associated with the destination set.
+     * In addition, updates the existing next-objective if new route-route paths found
+     * have resulted in the addition of new next-hops to a particular destination.
+     * If there is no existing next objective for this destination set, this method
+     * would create a next objective and return the nextId. Optionally metadata can be
      * passed in for the creation of the next objective.
      *
-     * @param ns neighborset
+     * @param ds destination set
+     * @param nextHops a map of per destination next hops
      * @param meta metadata passed into the creation of a Next Objective
      * @param isBos if Bos is set
      * @return int if found or -1 if there are errors in the creation of the
      *          neighbor set.
      */
-    public int getNextObjectiveId(NeighborSet ns, TrafficSelector meta, boolean isBos) {
-        Integer nextId = nsNextObjStore.
-                get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-        if (nextId == null) {
-            log.trace("getNextObjectiveId in device{}: Next objective id "
-                    + "not found for {} and creating", deviceId, ns);
+    public int getNextObjectiveId(DestinationSet ds,
+                                  Map<DeviceId, Set<DeviceId>> nextHops,
+                                  TrafficSelector meta, boolean isBos) {
+        NextNeighbors next = dsNextObjStore.
+                get(new DestinationSetNextObjectiveStoreKey(deviceId, ds));
+        if (next == null) {
+            log.debug("getNextObjectiveId in device{}: Next objective id "
+                    + "not found for {} ... creating", deviceId, ds);
             log.trace("getNextObjectiveId: nsNextObjStore contents for device {}: {}",
                       deviceId,
-                      nsNextObjStore.entrySet()
+                      dsNextObjStore.entrySet()
                       .stream()
                       .filter((nsStoreEntry) ->
                       (nsStoreEntry.getKey().deviceId().equals(deviceId)))
                       .collect(Collectors.toList()));
-            createGroupsFromNeighborsets(Collections.singleton(ns), meta, isBos);
-            nextId = nsNextObjStore.
-                    get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-            if (nextId == null) {
+
+            createGroupFromDestinationSet(ds, nextHops, meta, isBos);
+            next = dsNextObjStore.
+                    get(new DestinationSetNextObjectiveStoreKey(deviceId, ds));
+            if (next == null) {
                 log.warn("getNextObjectiveId: unable to create next objective");
+                // failure in creating group
                 return -1;
             } else {
                 log.debug("getNextObjectiveId in device{}: Next objective id {} "
-                    + "created for {}", deviceId, nextId, ns);
+                    + "created for {}", deviceId, next.nextId(), ds);
             }
         } else {
             log.trace("getNextObjectiveId in device{}: Next objective id {} "
-                    + "found for {}", deviceId, nextId, ns);
+                    + "found for {}", deviceId, next.nextId(), ds);
+            // should fix hash groups too if next-hops have changed
+            if (!next.dstNextHops().equals(nextHops)) {
+                log.debug("Nexthops have changed for dev:{} nextId:{} ..updating",
+                          deviceId, next.nextId());
+                if (!updateNextHops(ds, nextHops)) {
+                    // failure in updating group
+                    return -1;
+                }
+            }
         }
-        return nextId;
+        return next.nextId();
     }
 
     /**
@@ -663,10 +765,10 @@
      * @param ns neighbor set to check
      * @return true if it exists, false otherwise
      */
-    public boolean hasNextObjectiveId(NeighborSet ns) {
-        Integer nextId = nsNextObjStore.
-                get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-        if (nextId == null) {
+    public boolean hasNextObjectiveId(DestinationSet ns) {
+        NextNeighbors nextHops = dsNextObjStore.
+                get(new DestinationSetNextObjectiveStoreKey(deviceId, ns));
+        if (nextHops == null) {
             return false;
         }
 
@@ -704,100 +806,59 @@
         }
     }
 
-    protected Set<Set<DeviceId>> getPowerSetOfNeighbors(Set<DeviceId> neighbors) {
-        List<DeviceId> list = new ArrayList<>(neighbors);
-        Set<Set<DeviceId>> sets = new HashSet<>();
-        // get the number of elements in the neighbors
-        int elements = list.size();
-        // the number of members of a power set is 2^n
-        // including the empty set
-        int powerElements = (1 << elements);
-
-        // run a binary counter for the number of power elements
-        // NOTE: Exclude empty set
-        for (long i = 1; i < powerElements; i++) {
-            Set<DeviceId> neighborSubSet = new HashSet<>();
-            for (int j = 0; j < elements; j++) {
-                if ((i >> j) % 2 == 1) {
-                    neighborSubSet.add(list.get(j));
-                }
-            }
-            sets.add(neighborSubSet);
-        }
-        return sets;
-    }
-
-    private boolean isSegmentIdSameAsNodeSegmentId(DeviceId deviceId, int sId) {
-        int segmentId;
-        try {
-            // IPv6 sid is not inserted. this part of the code is not used for now.
-            segmentId = deviceConfig.getIPv4SegmentId(deviceId);
-        } catch (DeviceConfigNotFoundException e) {
-            log.warn(e.getMessage() + " Aborting isSegmentIdSameAsNodeSegmentId.");
-            return false;
-        }
-
-        return segmentId == sId;
-    }
-
-    protected List<Integer> getSegmentIdsTobePairedWithNeighborSet(Set<DeviceId> neighbors) {
-
-        List<Integer> nsSegmentIds = new ArrayList<>();
-
-        // Always pair up with no edge label
-        // If (neighbors.size() == 1) {
-        nsSegmentIds.add(-1);
-        // }
-
-        // Filter out SegmentIds matching with the
-        // nodes in the combo
-        for (Integer sId : allSegmentIds) {
-            if (sId.equals(this.ipv4NodeSegmentId)) {
-                continue;
-            }
-            boolean filterOut = false;
-            // Check if the edge label being set is of
-            // any node in the Neighbor set
-            for (DeviceId deviceId : neighbors) {
-                if (isSegmentIdSameAsNodeSegmentId(deviceId, sId)) {
-                    filterOut = true;
-                    break;
-                }
-            }
-            if (!filterOut) {
-                nsSegmentIds.add(sId);
-            }
-        }
-        return nsSegmentIds;
-    }
-
     /**
-     * Creates hash groups from a set of NeighborSet given.
+     * Creates a NextObjective for a hash group in this device from a given
+     * DestinationSet.
      *
-     * @param nsSet a set of NeighborSet
+     * @param ds the DestinationSet
+     * @param neighbors a map for each destination and its next-hops
      * @param meta metadata passed into the creation of a Next Objective
      * @param isBos if BoS is set
      */
-    public void createGroupsFromNeighborsets(Set<NeighborSet> nsSet,
-                                             TrafficSelector meta,
-                                             boolean isBos) {
-        for (NeighborSet ns : nsSet) {
-            int nextId = flowObjectiveService.allocateNextId();
-            NextObjective.Type type = NextObjective.Type.HASHED;
-            Set<DeviceId> neighbors = ns.getDeviceIds();
-            // If Bos == False and MPLS-ECMP == false, we have
-            // to use simple group and we will pick a single neighbor.
-            if (!isBos && !srManager.getMplsEcmp()) {
-                type = NextObjective.Type.SIMPLE;
-                neighbors = Collections.singleton(ns.getFirstNeighbor());
+    public void createGroupFromDestinationSet(DestinationSet ds,
+                                              Map<DeviceId, Set<DeviceId>> neighbors,
+                                              TrafficSelector meta,
+                                              boolean isBos) {
+        int nextId = flowObjectiveService.allocateNextId();
+        NextObjective.Type type = NextObjective.Type.HASHED;
+        if (neighbors == null || neighbors.isEmpty()) {
+            log.warn("createGroupsFromDestinationSet: needs at least one neighbor"
+                    + "to create group in dev:{} for ds: {} with next-hops {}",
+                    deviceId, ds, neighbors);
+            return;
+        }
+        // If Bos == False and MPLS-ECMP == false, we have
+        // to use simple group and we will pick a single neighbor for a single dest.
+        if (!isBos && !srManager.getMplsEcmp()) {
+            type = NextObjective.Type.SIMPLE;
+        }
+
+        NextObjective.Builder nextObjBuilder = DefaultNextObjective
+                .builder()
+                .withId(nextId)
+                .withType(type)
+                .fromApp(appId);
+        if (meta != null) {
+            nextObjBuilder.withMeta(meta);
+        }
+
+        // create treatment buckets for each neighbor for each dst Device
+        // except in the special case where we only want to pick a single
+        // neighbor for a simple group
+        boolean foundSingleNeighbor = false;
+        boolean treatmentAdded = false;
+        Map<DeviceId, Set<DeviceId>> dstNextHops = new ConcurrentHashMap<>();
+        for (DeviceId dst : ds.getDestinationSwitches()) {
+            Set<DeviceId> nextHops = neighbors.get(dst);
+            if (nextHops == null || nextHops.isEmpty()) {
+                continue;
             }
-            NextObjective.Builder nextObjBuilder = DefaultNextObjective
-                    .builder()
-                    .withId(nextId)
-                    .withType(type)
-                    .fromApp(appId);
-            // For each neighbor, we have to update the sent actions
-            for (DeviceId neighborId : neighbors) {
+
+            if (foundSingleNeighbor) {
+                break;
+            }
+
+            for (DeviceId neighborId : nextHops) {
                 if (devicePortMap.get(neighborId) == null) {
                     log.warn("Neighbor {} is not in the port map yet for dev:{}",
                              neighborId, deviceId);
@@ -812,53 +873,70 @@
                 try {
                     neighborMac = deviceConfig.getDeviceMac(neighborId);
                 } catch (DeviceConfigNotFoundException e) {
-                    log.warn(e.getMessage() + " Aborting createGroupsFromNeighborsets.");
+                    log.warn(e.getMessage() + " Aborting createGroupsFromDestinationset.");
                     return;
                 }
-                // For each port, we have to create a new treatment
+                // For each port to the neighbor, we create a new treatment
                 Set<PortNumber> neighborPorts = devicePortMap.get(neighborId);
                 // In this case we are using a SIMPLE group. We randomly pick a port
                 if (!isBos && !srManager.getMplsEcmp()) {
                     int size = devicePortMap.get(neighborId).size();
                     int index = RandomUtils.nextInt(0, size);
                     neighborPorts = Collections.singleton(
-                            Iterables.get(devicePortMap.get(neighborId), index)
-                    );
+                                        Iterables.get(devicePortMap.get(neighborId),
+                                                      index));
+                    foundSingleNeighbor = true;
                 }
                 for (PortNumber sp : neighborPorts) {
                     TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
                             .builder();
-                    tBuilder.setEthDst(neighborMac)
-                            .setEthSrc(nodeMacAddr);
-                    if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
+                    tBuilder.setEthDst(neighborMac).setEthSrc(nodeMacAddr);
+                    int edgeLabel = ds.getEdgeLabel(dst);
+                    if (edgeLabel != DestinationSet.NO_EDGE_LABEL) {
                         tBuilder.pushMpls()
-                                .copyTtlOut()
-                                .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
+                        .copyTtlOut()
+                        .setMpls(MplsLabel.mplsLabel(edgeLabel));
                     }
                     tBuilder.setOutput(sp);
                     nextObjBuilder.addTreatment(tBuilder.build());
+                    treatmentAdded = true;
+                    //update store
+                    Set<DeviceId> existingNeighbors = dstNextHops.get(dst);
+                    if (existingNeighbors == null) {
+                        existingNeighbors = new HashSet<>();
+                    }
+                    existingNeighbors.add(neighborId);
+                    dstNextHops.put(dst, existingNeighbors);
+                    log.debug("creating treatment for port/label {}/{} in next:{}",
+                              sp, edgeLabel, nextId);
+                }
+
+                if (foundSingleNeighbor) {
+                    break;
                 }
             }
-            if (meta != null) {
-                nextObjBuilder.withMeta(meta);
-            }
-
-            ObjectiveContext context = new DefaultObjectiveContext(
-                    (objective) ->
-                            log.debug("createGroupsFromNeighborsets installed "
-                                    + "NextObj {} on {}", nextId, deviceId),
-                    (objective, error) ->
-                            log.warn("createGroupsFromNeighborsets failed to install"
-                                    + " NextObj {} on {}: {}", nextId, deviceId, error)
-                    );
-            NextObjective nextObj = nextObjBuilder.add(context);
-            log.debug("**createGroupsFromNeighborsets: Submitted "
-                    + "next objective {} in device {}",
-                    nextId, deviceId);
-            flowObjectiveService.next(deviceId, nextObj);
-            nsNextObjStore.put(new NeighborSetNextObjectiveStoreKey(deviceId, ns),
-                               nextId);
         }
+
+        if (!treatmentAdded) {
+            log.warn("Could not createGroup from DestinationSet {} without any"
+                    + "next hops {}", ds, neighbors);
+            return;
+        }
+        ObjectiveContext context = new DefaultObjectiveContext(
+                (objective) ->
+                log.debug("createGroupsFromDestinationSet installed "
+                        + "NextObj {} on {}", nextId, deviceId),
+                (objective, error) ->
+                log.warn("createGroupsFromDestinationSet failed to install"
+                        + " NextObj {} on {}: {}", nextId, deviceId, error)
+                );
+        NextObjective nextObj = nextObjBuilder.add(context);
+        log.debug(".. createGroupsFromDestinationSet: Submitted "
+                + "next objective {} in device {}", nextId, deviceId);
+        flowObjectiveService.next(deviceId, nextObj);
+        //update store
+        dsNextObjStore.put(new DestinationSetNextObjectiveStoreKey(deviceId, ds),
+                           new NextNeighbors(dstNextHops, nextId));
     }
 
     /**
@@ -929,7 +1007,8 @@
      * @return true if the vlan id is not contained in any vlanTagged config
      */
     private boolean toPopVlan(PortNumber portNumber, VlanId vlanId) {
-        return srManager.interfaceService.getInterfacesByPort(new ConnectPoint(deviceId, portNumber))
+        return srManager.interfaceService
+                .getInterfacesByPort(new ConnectPoint(deviceId, portNumber))
                 .stream().noneMatch(intf -> intf.vlanTagged().contains(vlanId));
     }
 
@@ -977,8 +1056,11 @@
      * @return true if succeeds, false otherwise
      */
     public boolean removeGroup(int objectiveId) {
-
-        if (nsNextObjStore.containsValue(objectiveId)) {
+        for (Map.Entry<DestinationSetNextObjectiveStoreKey, NextNeighbors> e :
+                dsNextObjStore.entrySet()) {
+            if (e.getValue().nextId() != objectiveId) {
+                continue;
+            }
             NextObjective.Builder nextObjBuilder = DefaultNextObjective
                     .builder().withId(objectiveId)
                     .withType(NextObjective.Type.HASHED).fromApp(appId);
@@ -994,12 +1076,7 @@
                     objectiveId, deviceId);
             flowObjectiveService.next(deviceId, nextObjective);
 
-            for (Map.Entry<NeighborSetNextObjectiveStoreKey, Integer> entry: nsNextObjStore.entrySet()) {
-                if (entry.getValue().equals(objectiveId)) {
-                    nsNextObjStore.remove(entry.getKey());
-                    break;
-                }
-            }
+            dsNextObjStore.remove(e.getKey());
             return true;
         }
 
@@ -1009,10 +1086,10 @@
     /**
      * Removes all groups from all next objective stores.
      */
-    public void removeAllGroups() {
-        for (Map.Entry<NeighborSetNextObjectiveStoreKey, Integer> entry:
+    /*public void removeAllGroups() {
+        for (Map.Entry<NeighborSetNextObjectiveStoreKey, NextNeighbors> entry:
                 nsNextObjStore.entrySet()) {
-            removeGroup(entry.getValue());
+            removeGroup(entry.getValue().nextId());
         }
         for (Map.Entry<PortNextObjectiveStoreKey, Integer> entry:
                 portNextObjStore.entrySet()) {
@@ -1022,8 +1099,8 @@
                 vlanNextObjStore.entrySet()) {
             removeGroup(entry.getValue());
         }
-        // should probably clean local stores port-neighbor
-    }
+    }*/ //XXX revisit
+
 
     /**
      * RetryHashBkts is a one-time retry at populating all the buckets of a
@@ -1042,21 +1119,24 @@
         @Override
         public void run() {
             log.debug("RETRY Hash buckets for linkup: {}", link);
-            Set<NeighborSet> nsSet = nsNextObjStore.keySet()
+            Set<DestinationSetNextObjectiveStoreKey> dsKeySet = dsNextObjStore.entrySet()
                     .stream()
-                    .filter(nsStoreEntry -> nsStoreEntry.deviceId().equals(deviceId))
-                    .map(nsStoreEntry -> nsStoreEntry.neighborSet())
-                    .filter(ns -> ns.getDeviceIds()
-                            .contains(link.dst().deviceId()))
+                    .filter(entry -> entry.getKey().deviceId().equals(deviceId))
+                    .filter(entry -> entry.getValue().containsNextHop(link.dst().deviceId()))
+                    .map(entry -> entry.getKey())
                     .collect(Collectors.toSet());
-            log.debug("retry-link: nsNextObjStore contents for device {}: {}",
-                      deviceId, nsSet);
-            for (NeighborSet ns : nsSet) {
-                Integer nextId = nsNextObjStore.
-                        get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-                if (nextId != null) {
-                    addToHashedNextObjective(link.src().port(), dstMac, ns,
-                                             nextId, true);
+
+            log.debug("retry-link: dsNextObjStore contents for device {}: {}",
+                      deviceId, dsKeySet);
+            for (DestinationSetNextObjectiveStoreKey dsKey : dsKeySet) {
+                NextNeighbors next = dsNextObjStore.get(dsKey);
+                if (next != null) {
+                    Set<DeviceId> dstSet = next.getDstForNextHop(link.dst().deviceId());
+                    dstSet.forEach(dst -> {
+                        int edgeLabel = dsKey.destinationSet().getEdgeLabel(dst);
+                        addToHashedNextObjective(link.src().port(), dstMac, edgeLabel,
+                                                 next.nextId(), true);
+                    });
                 }
             }
         }
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DestinationSet.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DestinationSet.java
new file mode 100644
index 0000000..7b0864a
--- /dev/null
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DestinationSet.java
@@ -0,0 +1,239 @@
+/*
+ * Copyright 2015-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.segmentrouting.grouphandler;
+
+import org.onosproject.net.DeviceId;
+import org.slf4j.Logger;
+
+import com.google.common.base.MoreObjects.ToStringHelper;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Representation of a set of destination switch dpids along with their edge-node
+ * labels. Meant to be used as a lookup-key in a hash-map to retrieve an ECMP-group
+ * that hashes packets towards a specific destination switch,
+ * or paired-destination switches.
+ */
+public class DestinationSet {
+    public static final int NO_EDGE_LABEL = -1;
+    private static final int NOT_ASSIGNED = 0;
+    private boolean mplsSet;
+    private final DeviceId dstSw1;
+    private final int edgeLabel1;
+    private final DeviceId dstSw2;
+    private final int edgeLabel2;
+
+
+    protected static final Logger log = getLogger(DestinationSet.class);
+
+    /**
+     * Constructor for a single destination with no Edge label.
+     *
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param dstSw the destination switch
+     */
+    public DestinationSet(boolean isMplsSet, DeviceId dstSw) {
+        this.edgeLabel1 = NO_EDGE_LABEL;
+        this.mplsSet = isMplsSet;
+        this.dstSw1 = dstSw;
+        this.edgeLabel2 = NOT_ASSIGNED;
+        this.dstSw2 = null;
+    }
+
+    /**
+     * Constructor for a single destination with Edge label.
+     *
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param edgeLabel label to be pushed as part of group operation
+     * @param dstSw the destination switch
+     */
+    public DestinationSet(boolean isMplsSet,
+                       int edgeLabel, DeviceId dstSw) {
+        this.mplsSet = isMplsSet;
+        this.edgeLabel1 = edgeLabel;
+        this.dstSw1 = dstSw;
+        this.edgeLabel2 = NOT_ASSIGNED;
+        this.dstSw2 = null;
+    }
+
+    /**
+     * Constructor for paired destination switches and their associated
+     * edge labels.
+     *
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param edgeLabel1 label to be pushed as part of group operation for dstSw1
+     * @param dstSw1 one of the paired destination switches
+     * @param edgeLabel2 label to be pushed as part of group operation for dstSw2
+     * @param dstSw2 the other paired destination switch
+     */
+    public DestinationSet(boolean isMplsSet,
+                          int edgeLabel1, DeviceId dstSw1,
+                          int edgeLabel2, DeviceId dstSw2) {
+        this.mplsSet = isMplsSet;
+        if (dstSw1.toString().compareTo(dstSw2.toString()) <= 0) {
+            this.edgeLabel1 = edgeLabel1;
+            this.dstSw1 = dstSw1;
+            this.edgeLabel2 = edgeLabel2;
+            this.dstSw2 = dstSw2;
+        } else {
+            this.edgeLabel1 = edgeLabel2;
+            this.dstSw1 = dstSw2;
+            this.edgeLabel2 = edgeLabel1;
+            this.dstSw2 = dstSw1;
+        }
+    }
+
+    /**
+     * Default constructor for kryo serialization.
+     */
+    public DestinationSet() {
+        this.edgeLabel1 = NOT_ASSIGNED;
+        this.edgeLabel2 = NOT_ASSIGNED;
+        this.mplsSet = true;
+        this.dstSw1 = DeviceId.NONE;
+        this.dstSw2 = DeviceId.NONE;
+    }
+
+    /**
+     * Factory method for DestinationSet hierarchy.
+     *
+     * @param random the expected behavior.
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param dstSw the destination switch
+     * @return the destination set object.
+     */
+    public static DestinationSet destinationSet(boolean random,
+                                          boolean isMplsSet, DeviceId dstSw) {
+        return random ? new RandomDestinationSet(dstSw)
+                      : new DestinationSet(isMplsSet, dstSw);
+    }
+
+    /**
+     * Factory method for DestinationSet hierarchy.
+     *
+     * @param random the expected behavior.
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param edgeLabel label to be pushed as part of group operation
+     * @param dstSw the destination switch
+     * @return the destination set object
+     */
+    public static DestinationSet destinationSet(boolean random,
+                                          boolean isMplsSet, int edgeLabel,
+                                          DeviceId dstSw) {
+        return random ? new RandomDestinationSet(edgeLabel, dstSw)
+                      : new DestinationSet(isMplsSet, edgeLabel, dstSw);
+    }
+
+    /**
+     * Factory method for DestinationSet hierarchy.
+     *
+     * @param random the expected behavior.
+     * @return the destination set object
+     */
+    public static DestinationSet destinationSet(boolean random) {
+        return random ? new RandomDestinationSet() : new DestinationSet();
+    }
+
+    /**
+     * Gets the label associated with given destination switch.
+     *
+     * @param dstSw the destination switch
+     * @return integer the label associated with the destination switch
+     */
+    public int getEdgeLabel(DeviceId dstSw) {
+        if (dstSw.equals(dstSw1)) {
+            return edgeLabel1;
+        } else if (dstSw.equals(dstSw2)) {
+            return edgeLabel2;
+        }
+        return NOT_ASSIGNED;
+    }
+
+    /**
+     * Gets all the destination switches in this destination set.
+     *
+     * @return a set of destination switch ids
+     */
+    public Set<DeviceId> getDestinationSwitches() {
+        Set<DeviceId> dests = new HashSet<>();
+        dests.add(dstSw1);
+        if (dstSw2 != null) {
+            dests.add(dstSw2);
+        }
+        return dests;
+    }
+
+    /**
+     * Gets the value of mplsSet.
+     *
+     * @return the value of mplsSet
+     */
+    public boolean mplsSet() {
+        return mplsSet;
+    }
+
+    // The list of destination ids and label are used for comparison.
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof DestinationSet)) {
+            return false;
+        }
+        DestinationSet that = (DestinationSet) o;
+        boolean equal = (this.edgeLabel1 == that.edgeLabel1 &&
+                            this.mplsSet == that.mplsSet &&
+                            this.dstSw1.equals(that.dstSw1));
+        if (this.dstSw2 != null && that.dstSw2 == null ||
+                this.dstSw2 == null && that.dstSw2 != null) {
+            return false;
+        }
+        if (this.dstSw2 != null && that.dstSw2 != null) {
+            equal = equal && (this.edgeLabel2 == that.edgeLabel2 &&
+                                this.dstSw2.equals(that.dstSw2));
+        }
+        return equal;
+    }
+
+    // The list of destination ids and label are used for comparison.
+    @Override
+    public int hashCode() {
+        if (dstSw2 == null) {
+            return Objects.hash(mplsSet, edgeLabel1, dstSw1);
+        }
+        return Objects.hash(mplsSet, edgeLabel1, dstSw1, edgeLabel2, dstSw2);
+    }
+
+    @Override
+    public String toString() {
+        ToStringHelper h = toStringHelper(this)
+                                .add("MplsSet", mplsSet)
+                                .add("DstSw1", dstSw1)
+                                .add("Label1", edgeLabel1);
+        if (dstSw2 != null) {
+            h.add("DstSw2", dstSw2)
+             .add("Label2", edgeLabel2);
+        }
+        return h.toString();
+    }
+}
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NeighborSet.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NeighborSet.java
deleted file mode 100644
index 88799f4..0000000
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NeighborSet.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Copyright 2015-present Open Networking Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.onosproject.segmentrouting.grouphandler;
-
-import org.onosproject.net.DeviceId;
-import org.slf4j.Logger;
-
-import java.util.HashSet;
-import java.util.Objects;
-import java.util.Set;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.slf4j.LoggerFactory.getLogger;
-
-/**
- * Representation of a set of neighbor switch dpids along with edge node
- * label and a destination switch. Meant to be used as a lookup-key in a hash-map
- * to retrieve an ECMP-group that hashes packets to a set of ports connecting to
- * the neighbors in this set towards a specific destination switch.
- */
-public class NeighborSet {
-    private final Set<DeviceId> neighbors;
-    private final int edgeLabel;
-    public static final int NO_EDGE_LABEL = -1;
-    private boolean mplsSet;
-    // the destination switch towards which the neighbors are the next-hops.
-    private final DeviceId dstSw;
-    protected static final Logger log = getLogger(NeighborSet.class);
-
-    /**
-     * Constructor with set of neighbors. Edge label is
-     * default to -1.
-     *
-     * @param neighbors set of neighbors representing the next-hops
-     * @param isMplsSet indicates if it is a mpls neighbor set
-     * @param dstSw the destination switch
-     */
-    public NeighborSet(Set<DeviceId> neighbors, boolean isMplsSet, DeviceId dstSw) {
-        checkNotNull(neighbors);
-        this.edgeLabel = NO_EDGE_LABEL;
-        this.neighbors = new HashSet<>();
-        this.neighbors.addAll(neighbors);
-        this.mplsSet = isMplsSet;
-        this.dstSw = dstSw;
-    }
-
-    /**
-     * Constructor with set of neighbors and edge label.
-     *
-     * @param neighbors set of neighbors representing the next-hops
-     * @param isMplsSet indicates if it is a mpls neighbor set
-     * @param edgeLabel label to be pushed as part of group operation
-     * @param dstSw the destination switch
-     */
-    public NeighborSet(Set<DeviceId> neighbors, boolean isMplsSet,
-                       int edgeLabel, DeviceId dstSw) {
-        checkNotNull(neighbors);
-        this.edgeLabel = edgeLabel;
-        this.neighbors = new HashSet<>();
-        this.neighbors.addAll(neighbors);
-        this.mplsSet = isMplsSet;
-        this.dstSw = dstSw;
-    }
-
-    /**
-     * Default constructor for kryo serialization.
-     */
-    public NeighborSet() {
-        this.edgeLabel = NO_EDGE_LABEL;
-        this.neighbors = new HashSet<>();
-        this.mplsSet = true;
-        this.dstSw = DeviceId.NONE;
-    }
-
-    /**
-     * Factory method for NeighborSet hierarchy.
-     *
-     * @param random the expected behavior.
-     * @param neighbors the set of neighbors representing the next-hops
-     * @param isMplsSet indicates if it is a mpls neighbor set
-     * @param dstSw the destination switch
-     * @return the neighbor set object.
-     */
-    public static NeighborSet neighborSet(boolean random, Set<DeviceId> neighbors,
-                                          boolean isMplsSet, DeviceId dstSw) {
-        return random ? new RandomNeighborSet(neighbors, dstSw)
-                      : new NeighborSet(neighbors, isMplsSet, dstSw);
-    }
-
-    /**
-     * Factory method for NeighborSet hierarchy.
-     *
-     * @param random the expected behavior.
-     * @param neighbors the set of neighbors representing the next-hops
-     * @param isMplsSet indicates if it is a mpls neighbor set
-     * @param edgeLabel label to be pushed as part of group operation
-     * @param dstSw the destination switch
-     * @return the neighbor set object
-     */
-    public static NeighborSet neighborSet(boolean random, Set<DeviceId> neighbors,
-                                          boolean isMplsSet, int edgeLabel,
-                                          DeviceId dstSw) {
-        return random ? new RandomNeighborSet(neighbors, edgeLabel, dstSw)
-                      : new NeighborSet(neighbors, isMplsSet, edgeLabel, dstSw);
-    }
-
-    /**
-     * Factory method for NeighborSet hierarchy.
-     *
-     * @param random the expected behavior.
-     * @return the neighbor set object
-     */
-    public static NeighborSet neighborSet(boolean random) {
-        return random ? new RandomNeighborSet() : new NeighborSet();
-    }
-
-    /**
-     * Gets the neighbors part of neighbor set.
-     *
-     * @return set of neighbor identifiers
-     */
-    public Set<DeviceId> getDeviceIds() {
-        return neighbors;
-    }
-
-    /**
-     * Gets the label associated with neighbor set.
-     *
-     * @return integer
-     */
-    public int getEdgeLabel() {
-        return edgeLabel;
-    }
-
-    /**
-     * Gets the destination switch for this neighbor set.
-     *
-     * @return the destination switch id
-     */
-    public DeviceId getDestinationSw() {
-        return dstSw;
-    }
-
-    /**
-     * Gets the first neighbor of the set. The default
-     * implementation assure the first neighbor is the
-     * first of the set. Subclasses can modify this.
-     *
-     * @return the first neighbor of the set
-     */
-    public DeviceId getFirstNeighbor() {
-        return neighbors.isEmpty() ? DeviceId.NONE : neighbors.iterator().next();
-    }
-
-    /**
-     * Gets the value of mplsSet.
-     *
-     * @return the value of mplsSet
-     */
-    public boolean mplsSet() {
-        return mplsSet;
-    }
-
-    // The list of neighbor ids and label are used for comparison.
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (!(o instanceof NeighborSet)) {
-            return false;
-        }
-        NeighborSet that = (NeighborSet) o;
-        return (this.neighbors.containsAll(that.neighbors) &&
-                that.neighbors.containsAll(this.neighbors) &&
-                this.edgeLabel == that.edgeLabel &&
-                this.mplsSet == that.mplsSet &&
-                this.dstSw.equals(that.dstSw));
-    }
-
-    // The list of neighbor ids and label are used for comparison.
-    @Override
-    public int hashCode() {
-        return Objects.hash(neighbors, edgeLabel, mplsSet, dstSw);
-    }
-
-    @Override
-    public String toString() {
-        return toStringHelper(this)
-                .add("Neighbors", neighbors)
-                .add("Label", edgeLabel)
-                .add("MplsSet", mplsSet)
-                .add("DstSw", dstSw)
-                .toString();
-    }
-}
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NextNeighbors.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NextNeighbors.java
new file mode 100644
index 0000000..1a0507b
--- /dev/null
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NextNeighbors.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2015-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.segmentrouting.grouphandler;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import org.onosproject.net.DeviceId;
+
+public class NextNeighbors {
+    private final Map<DeviceId, Set<DeviceId>> dstNextHops;
+    private final int nextId;
+
+    public NextNeighbors(Map<DeviceId, Set<DeviceId>> dstNextHops, int nextId) {
+        this.dstNextHops = dstNextHops;
+        this.nextId = nextId;
+    }
+
+    public Map<DeviceId, Set<DeviceId>> dstNextHops() {
+        return dstNextHops;
+    }
+
+    public Set<DeviceId> nextHops(DeviceId deviceId) {
+        return dstNextHops.get(deviceId);
+    }
+
+    public int nextId() {
+        return nextId;
+    }
+
+    public boolean containsNextHop(DeviceId nextHopId) {
+        for (Set<DeviceId> nextHops : dstNextHops.values()) {
+            if (nextHops != null && nextHops.contains(nextHopId)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public Set<DeviceId> getDstForNextHop(DeviceId nextHopId) {
+        Set<DeviceId> dstSet = new HashSet<>();
+        for (DeviceId dstKey : dstNextHops.keySet()) {
+            if (dstNextHops.get(dstKey).contains(nextHopId)) {
+                dstSet.add(dstKey);
+            }
+        }
+        return dstSet;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof NextNeighbors)) {
+            return false;
+        }
+        NextNeighbors that = (NextNeighbors) o;
+        return (this.nextId == that.nextId) &&
+                this.dstNextHops.equals(that.dstNextHops);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nextId, dstNextHops);
+    }
+
+    @Override
+    public String toString() {
+        return toStringHelper(this)
+                .add("nextId", nextId)
+                .add("dstNextHops", dstNextHops)
+                .toString();
+    }
+}
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
index 2ba7c19..c0935ce 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
@@ -199,7 +199,7 @@
                     tBuilder.setOutput(bucketId.outPort())
                             .setEthDst(neighborEthDst)
                             .setEthSrc(nodeMacAddr);
-                    if (bucketId.label() != NeighborSet.NO_EDGE_LABEL) {
+                    if (bucketId.label() != DestinationSet.NO_EDGE_LABEL) {
                         tBuilder.pushMpls()
                             .setMpls(MplsLabel.mplsLabel(bucketId.label()));
                     }
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomNeighborSet.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomDestinationSet.java
similarity index 66%
rename from apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomNeighborSet.java
rename to apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomDestinationSet.java
index edf178f..3671b03 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomNeighborSet.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomDestinationSet.java
@@ -16,32 +16,32 @@
 
 package org.onosproject.segmentrouting.grouphandler;
 
-import com.google.common.collect.Iterables;
-import org.apache.commons.lang3.RandomUtils;
 import org.onosproject.net.DeviceId;
 
-import java.util.Set;
-
 /**
  * Extends its super class modifying its internal behavior.
  * Pick a neighbor will pick a random neighbor.
  */
-public class RandomNeighborSet extends NeighborSet {
+public class RandomDestinationSet extends DestinationSet {
 
-    public RandomNeighborSet(Set<DeviceId> neighbors, DeviceId dstSw) {
-        super(neighbors, true, dstSw);
+    public RandomDestinationSet(DeviceId dstSw) {
+        super(true, dstSw);
     }
 
-    public RandomNeighborSet(Set<DeviceId> neighbors, int edgeLabel,
+    public RandomDestinationSet(int edgeLabel,
                              DeviceId dstSw) {
-        super(neighbors, true, edgeLabel, dstSw);
+        super(true, edgeLabel, dstSw);
     }
 
-    public RandomNeighborSet() {
+    public RandomDestinationSet() {
         super();
     }
 
-    @Override
+    // XXX revisit the need for this class since neighbors no longer stored here
+    // will be handled when we fix pseudowires for dual-Tor scenarios
+
+
+    /*@Override
     public DeviceId getFirstNeighbor() {
         if (getDeviceIds().isEmpty()) {
             return DeviceId.NONE;
@@ -49,12 +49,12 @@
         int size = getDeviceIds().size();
         int index = RandomUtils.nextInt(0, size);
         return Iterables.get(getDeviceIds(), index);
-    }
+    }*/
 
     @Override
     public String toString() {
-        return " RandomNeighborset Sw: " + getDeviceIds()
-                + " and Label: " + getEdgeLabel()
-                + " for destination: " + getDestinationSw();
+        return " RandomNeighborset Sw: " //+ getDeviceIds()
+                + " and Label: " //+ getEdgeLabel()
+                + " for destination: "; // + getDestinationSw();
     }
 }