diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
index 9b27164..bcd6a75 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
@@ -25,6 +25,7 @@
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip6Address;
 import org.onlab.packet.IpPrefix;
+import org.onosproject.cluster.NodeId;
 import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.Device;
 import org.onosproject.net.DeviceId;
@@ -38,6 +39,8 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ScheduledExecutorService;
@@ -58,7 +61,6 @@
     private static final int MAX_CONSTANT_RETRY_ATTEMPTS = 5;
     private static final int RETRY_INTERVAL_MS = 250;
     private static final int RETRY_INTERVAL_SCALE = 1;
-    private static final String ECMPSPG_MISSING = "ECMP shortest path graph not found";
     private static Logger log = LoggerFactory.getLogger(DefaultRoutingHandler.class);
 
     private SegmentRoutingManager srManager;
@@ -106,7 +108,7 @@
      * Returns an immutable copy of the current ECMP shortest-path graph as
      * computed by this controller instance.
      *
-     * @return the current ECMP graph
+     * @return immutable copy of the current ECMP graph
      */
     public ImmutableMap<DeviceId, EcmpShortestPathGraph> getCurrentEmcpSpgMap() {
         Builder<DeviceId, EcmpShortestPathGraph> builder = ImmutableMap.builder();
@@ -118,42 +120,76 @@
         return builder.build();
     }
 
+    //////////////////////////////////////
+    //  Route path handling
+    //////////////////////////////////////
+
+    /* The following three methods represent the three major ways in routing
+     * is triggered in the network
+     *      a) due to configuration change
+     *      b) due to route-added event
+     *      c) due to change in the topology
+     */
+
     /**
-     * Populates all routing rules to all connected routers, including default
-     * routing rules, adjacency rules, and policy rules if any.
+     * Populates all routing rules to all switches. Typically triggered at
+     * startup or after a configuration event.
      */
     public void populateAllRoutingRules() {
-
         statusLock.lock();
         try {
+            if (populationStatus == Status.STARTED) {
+                log.warn("Previous rule population is not finished. Cannot"
+                        + " proceed with populateAllRoutingRules");
+                return;
+            }
+
             populationStatus = Status.STARTED;
             rulePopulator.resetCounter();
-            log.info("Starting to populate segment-routing rules");
+            log.info("Starting to populate all routing rules");
             log.debug("populateAllRoutingRules: populationStatus is STARTED");
 
-            for (Device sw : srManager.deviceService.getDevices()) {
-                if (!srManager.mastershipService.isLocalMaster(sw.id())) {
-                    log.debug("populateAllRoutingRules: skipping device {}..."
-                            + "we are not master", sw.id());
+            // take a snapshot of the topology
+            updatedEcmpSpgMap = new HashMap<>();
+            Set<EdgePair> edgePairs = new HashSet<>();
+            Set<ArrayList<DeviceId>> routeChanges = new HashSet<>();
+            for (Device dstSw : srManager.deviceService.getDevices()) {
+                EcmpShortestPathGraph ecmpSpgUpdated =
+                        new EcmpShortestPathGraph(dstSw.id(), srManager);
+                updatedEcmpSpgMap.put(dstSw.id(), ecmpSpgUpdated);
+                DeviceId pairDev = getPairDev(dstSw.id());
+                if (pairDev != null) {
+                    // pairDev may not be available yet, but we still need to add
+                    ecmpSpgUpdated = new EcmpShortestPathGraph(pairDev, srManager);
+                    updatedEcmpSpgMap.put(pairDev, ecmpSpgUpdated);
+                    edgePairs.add(new EdgePair(dstSw.id(), pairDev));
+                }
+                DeviceId ret = shouldHandleRouting(dstSw.id());
+                if (ret == null) {
                     continue;
                 }
-
-                EcmpShortestPathGraph ecmpSpg = new EcmpShortestPathGraph(sw.id(), srManager);
-                if (!populateEcmpRoutingRules(sw.id(), ecmpSpg, ImmutableSet.of())) {
-                    log.debug("populateAllRoutingRules: populationStatus is ABORTED");
-                    populationStatus = Status.ABORTED;
-                    log.debug("Abort routing rule population");
-                    return;
+                Set<DeviceId> devsToProcess = Sets.newHashSet(dstSw.id(), ret);
+                // To do a full reroute, assume all routes have changed
+                for (DeviceId dev : devsToProcess) {
+                    for (Device targetSw : srManager.deviceService.getDevices()) {
+                        if (targetSw.id().equals(dev)) {
+                            continue;
+                        }
+                        routeChanges.add(Lists.newArrayList(targetSw.id(), dev));
+                    }
                 }
-                currentEcmpSpgMap.put(sw.id(), ecmpSpg);
-                log.debug("Updating ECMPspg for sw:{}", sw.id());
+            }
 
-                // TODO: Set adjacency routing rule for all switches
+            if (!redoRouting(routeChanges, edgePairs, null)) {
+                log.debug("populateAllRoutingRules: populationStatus is ABORTED");
+                populationStatus = Status.ABORTED;
+                log.warn("Failed to repopulate all routing rules.");
+                return;
             }
 
             log.debug("populateAllRoutingRules: populationStatus is SUCCEEDED");
             populationStatus = Status.SUCCEEDED;
-            log.info("Completed routing rule population. Total # of rules pushed : {}",
+            log.info("Completed all routing rule population. Total # of rules pushed : {}",
                     rulePopulator.getCounter());
             return;
         } finally {
@@ -162,6 +198,114 @@
     }
 
     /**
+     * Populate rules from all other edge devices to the connect-point(s)
+     * specified for the given subnets.
+     *
+     * @param cpts connect point(s) of the subnets being added
+     * @param subnets subnets being added
+     */ //XXX refactor
+    protected void populateSubnet(Set<ConnectPoint> cpts, Set<IpPrefix> subnets) {
+        statusLock.lock();
+        try {
+           if (populationStatus == Status.STARTED) {
+                log.warn("Previous rule population is not finished. Cannot"
+                        + " proceed with routing rules for added routes");
+                return;
+            }
+            populationStatus = Status.STARTED;
+            rulePopulator.resetCounter();
+            log.info("Starting to populate routing rules for added routes");
+            // Take snapshots of the topology
+            updatedEcmpSpgMap = new HashMap<>();
+            Set<EdgePair> edgePairs = new HashSet<>();
+            Set<ArrayList<DeviceId>> routeChanges = new HashSet<>();
+            boolean handleRouting = false;
+
+            if (cpts.size() == 2) {
+                // ensure connect points are edge-pairs
+                Iterator<ConnectPoint> iter = cpts.iterator();
+                DeviceId dev1 = iter.next().deviceId();
+                DeviceId pairDev = getPairDev(dev1);
+                if (iter.next().deviceId().equals(pairDev)) {
+                    edgePairs.add(new EdgePair(dev1, pairDev));
+                } else {
+                    log.warn("Connectpoints {} for subnets {} not on "
+                            + "pair-devices.. aborting populateSubnet", cpts, subnets);
+                    populationStatus = Status.ABORTED;
+                    return;
+                }
+                for (ConnectPoint cp : cpts) {
+                    EcmpShortestPathGraph ecmpSpgUpdated =
+                            new EcmpShortestPathGraph(cp.deviceId(), srManager);
+                    updatedEcmpSpgMap.put(cp.deviceId(), ecmpSpgUpdated);
+                    DeviceId retId = shouldHandleRouting(cp.deviceId());
+                    if (retId == null) {
+                        continue;
+                    }
+                    handleRouting = true;
+                }
+            } else {
+                // single connect point
+                DeviceId dstSw = cpts.iterator().next().deviceId();
+                EcmpShortestPathGraph ecmpSpgUpdated =
+                        new EcmpShortestPathGraph(dstSw, srManager);
+                updatedEcmpSpgMap.put(dstSw, ecmpSpgUpdated);
+                if (srManager.mastershipService.isLocalMaster(dstSw)) {
+                    handleRouting = true;
+                }
+            }
+
+            if (!handleRouting) {
+                log.debug("This instance is not handling ecmp routing to the "
+                        + "connectPoint(s) {}", cpts);
+                populationStatus = Status.ABORTED;
+                return;
+            }
+
+            // if it gets here, this instance should handle routing for the
+            // connectpoint(s). Assume all route-paths have to be updated to
+            // the connectpoint(s) with the following exceptions
+            // 1. if target is non-edge no need for routing rules
+            // 2. if target is one of the connectpoints
+            for (ConnectPoint cp : cpts) {
+                DeviceId dstSw = cp.deviceId();
+                for (Device targetSw : srManager.deviceService.getDevices()) {
+                    boolean isEdge = false;
+                    try {
+                        isEdge = config.isEdgeDevice(targetSw.id());
+                    } catch (DeviceConfigNotFoundException e) {
+                        log.warn(e.getMessage() + "aborting populateSubnet");
+                        populationStatus = Status.ABORTED;
+                        return;
+                    }
+                    if (dstSw.equals(targetSw.id()) || !isEdge ||
+                            (cpts.size() == 2 &&
+                                targetSw.id().equals(getPairDev(dstSw)))) {
+                        continue;
+                    }
+                    routeChanges.add(Lists.newArrayList(targetSw.id(), dstSw));
+                }
+            }
+
+            if (!redoRouting(routeChanges, edgePairs, subnets)) {
+                log.debug("populateSubnet: populationStatus is ABORTED");
+                populationStatus = Status.ABORTED;
+                log.warn("Failed to repopulate the rules for subnet.");
+                return;
+            }
+
+            log.debug("populateSubnet: populationStatus is SUCCEEDED");
+            populationStatus = Status.SUCCEEDED;
+            log.info("Completed subnet population. Total # of rules pushed : {}",
+                    rulePopulator.getCounter());
+            return;
+
+        } finally {
+            statusLock.unlock();
+        }
+    }
+
+    /**
      * Populates the routing rules or makes hash group changes according to the
      * route-path changes due to link failure, switch failure or link up. This
      * method should only be called for one of these three possible event-types.
@@ -174,7 +318,7 @@
      *                  link-down or a removed switch
      * @param switchDown the removed switch, or null for other conditions such as
      *                  link-down or link-up
-     */
+     */ // refactor
     public void populateRoutingRulesForLinkStatusChange(Link linkDown,
                                                            Link linkUp,
                                                            DeviceId switchDown) {
@@ -189,27 +333,34 @@
         try {
 
             if (populationStatus == Status.STARTED) {
-                log.warn("Previous rule population is not finished.");
+                log.warn("Previous rule population is not finished. Cannot"
+                        + " proceeed with routingRules for Link Status change");
                 return;
             }
 
-            // Take the snapshots of the links
+            // Take snapshots of the topology
             updatedEcmpSpgMap = new HashMap<>();
+            Set<EdgePair> edgePairs = new HashSet<>();
             for (Device sw : srManager.deviceService.getDevices()) {
-                if (!srManager.mastershipService.isLocalMaster(sw.id())) {
-                    continue;
-                }
                 EcmpShortestPathGraph ecmpSpgUpdated =
                         new EcmpShortestPathGraph(sw.id(), srManager);
                 updatedEcmpSpgMap.put(sw.id(), ecmpSpgUpdated);
+                DeviceId pairDev = getPairDev(sw.id());
+                if (pairDev != null) {
+                    // pairDev may not be available yet, but we still need to add
+                    ecmpSpgUpdated = new EcmpShortestPathGraph(pairDev, srManager);
+                    updatedEcmpSpgMap.put(pairDev, ecmpSpgUpdated);
+                    edgePairs.add(new EdgePair(sw.id(), pairDev));
+                }
             }
 
-            log.info("Starts rule population from link change");
+            log.info("Starting to populate routing rules from link status change");
 
             Set<ArrayList<DeviceId>> routeChanges;
             log.debug("populateRoutingRulesForLinkStatusChange: "
                     + "populationStatus is STARTED");
             populationStatus = Status.STARTED;
+            rulePopulator.resetCounter();
             // try optimized re-routing
             if (linkDown == null) {
                 // either a linkUp or a switchDown - compute all route changes by
@@ -219,7 +370,7 @@
                 if (routeChanges != null) {
                     // deal with linkUp of a seen-before link
                     if (linkUp != null && srManager.isSeenLink(linkUp)) {
-                        if (!isBidirectional(linkUp)) {
+                        if (!srManager.isBidirectional(linkUp)) {
                             log.warn("Not a bidirectional link yet .. not "
                                     + "processing link {}", linkUp);
                             srManager.updateSeenLink(linkUp, true);
@@ -264,7 +415,9 @@
 
             // do full re-routing if optimized routing returns null routeChanges
             if (routeChanges == null) {
-                log.info("Optimized routing failed... doing full re-route");
+                log.info("Optimized routing failed... opting for full reroute");
+                populationStatus = Status.ABORTED;
+                statusLock.unlock();
                 populateAllRoutingRules();
                 return;
             }
@@ -277,16 +430,16 @@
             }
 
             // reroute of routeChanges
-            if (repopulateRoutingRulesForRoutes(routeChanges)) {
+            if (redoRouting(routeChanges, edgePairs, null)) {
                 log.debug("populateRoutingRulesForLinkStatusChange: populationStatus is SUCCEEDED");
                 populationStatus = Status.SUCCEEDED;
-                log.info("Completed repopulation of rules. # of rules populated : {}",
-                        rulePopulator.getCounter());
+                log.info("Completed repopulation of rules for link-status change."
+                        + " # of rules populated : {}", rulePopulator.getCounter());
                 return;
             } else {
                 log.debug("populateRoutingRulesForLinkStatusChange: populationStatus is ABORTED");
                 populationStatus = Status.ABORTED;
-                log.warn("Failed to repopulate the rules.");
+                log.warn("Failed to repopulate the rules for link status change.");
                 return;
             }
         } finally {
@@ -295,24 +448,354 @@
     }
 
     /**
-     * Returns true if the link being queried is a bidirectional link. A bidi
-     * link is defined as a link, whose reverse link - ie. the link in the reverse
-     * direction - has been seen-before and is up.
+     * Processes a set a route-path changes by reprogramming routing rules and
+     * creating new hash-groups or editing them if necessary. This method also
+     * determines the next-hops for the route-path from the src-switch (target)
+     * of the path towards the dst-switch of the path.
      *
-     * @param link the infrastructure link being queried
-     * @return true if another unidirectional link exists in the reverse direction,
-     *              has been seen-before and is up
+     * @param routeChanges a set of route-path changes, where each route-path is
+     *                     a list with its first element the src-switch (target)
+     *                     of the path, and the second element the dst-switch of
+     *                     the path.
+     * @param edgePairs a set of edge-switches that are paired by configuration
+     * @param subnets  a set of prefixes that need to be populated in the routing
+     *                 table of the target switch in the route-path. Can be null,
+     *                 in which case all the prefixes belonging to the dst-switch
+     *                 will be populated in the target switch
+     * @return true if successful in repopulating all routes
      */
-    private boolean isBidirectional(Link link) {
-        Link reverseLink = srManager.linkService.getLink(link.dst(), link.src());
-        if (reverseLink == null) {
+    private boolean redoRouting(Set<ArrayList<DeviceId>> routeChanges,
+                                Set<EdgePair> edgePairs, Set<IpPrefix> subnets) {
+        // first make every entry two-elements
+        Set<ArrayList<DeviceId>> changedRoutes = new HashSet<>();
+        for (ArrayList<DeviceId> route : routeChanges) {
+            if (route.size() == 1) {
+                DeviceId dstSw = route.get(0);
+                EcmpShortestPathGraph ec = updatedEcmpSpgMap.get(dstSw);
+                if (ec == null) {
+                    log.warn("No graph found for {} .. aborting redoRouting", dstSw);
+                    return false;
+                }
+                ec.getAllLearnedSwitchesAndVia().keySet().forEach(key -> {
+                    ec.getAllLearnedSwitchesAndVia().get(key).keySet().forEach(target -> {
+                        changedRoutes.add(Lists.newArrayList(target, dstSw));
+                    });
+                });
+            } else {
+                DeviceId targetSw = route.get(0);
+                DeviceId dstSw = route.get(1);
+                changedRoutes.add(Lists.newArrayList(targetSw, dstSw));
+            }
+        }
+
+        // now process changedRoutes according to edgePairs
+        if (!redoRoutingEdgePairs(edgePairs, subnets, changedRoutes)) {
+            return false; //abort routing and fail fast
+        }
+
+        // whatever is left in changedRoutes is now processed for individual dsts.
+        if (!redoRoutingIndividualDests(subnets, changedRoutes)) {
+            return false; //abort routing and fail fast
+        }
+
+        //XXX should we do hashgroupchanges here?
+
+        // update ecmpSPG for all edge-pairs
+        for (EdgePair ep : edgePairs) {
+            currentEcmpSpgMap.put(ep.dev1, updatedEcmpSpgMap.get(ep.dev1));
+            currentEcmpSpgMap.put(ep.dev2, updatedEcmpSpgMap.get(ep.dev2));
+            log.debug("Updating ECMPspg for edge-pair:{}-{}", ep.dev1, ep.dev2);
+        }
+        return true;
+    }
+
+    /**
+     * Programs targetSw in the changedRoutes for given prefixes reachable by
+     * an edgePair. If no prefixes are given, the method will use configured
+     * subnets/prefixes. If some configured subnets belong only to a specific
+     * destination in the edgePair, then the target switch will be programmed
+     * only to that destination.
+     *
+     * @param edgePairs set of edge-pairs for which target will be programmed
+     * @param subnets a set of prefixes that need to be populated in the routing
+     *                 table of the target switch in the changedRoutes. Can be null,
+     *                 in which case all the configured prefixes belonging to the
+     *                 paired switches will be populated in the target switch
+     * @param changedRoutes a set of route-path changes, where each route-path is
+     *                     a list with its first element the src-switch (target)
+     *                     of the path, and the second element the dst-switch of
+     *                     the path.
+     * @return true if successful
+     */
+    private boolean redoRoutingEdgePairs(Set<EdgePair> edgePairs,
+                                      Set<IpPrefix> subnets,
+                                      Set<ArrayList<DeviceId>> changedRoutes) {
+        for (EdgePair ep : edgePairs) {
+            // temp store for a target's changedRoutes to this edge-pair
+            Map<DeviceId, Set<ArrayList<DeviceId>>> targetRoutes = new HashMap<>();
+            Iterator<ArrayList<DeviceId>> i = changedRoutes.iterator();
+            while (i.hasNext()) {
+                ArrayList<DeviceId> route = i.next();
+                DeviceId dstSw = route.get(1);
+                if (ep.includes(dstSw)) {
+                    // routeChange for edge pair found
+                    // sort by target iff target is edge and remove from changedRoutes
+                    DeviceId targetSw = route.get(0);
+                    try {
+                        if (!srManager.deviceConfiguration.isEdgeDevice(targetSw)) {
+                            continue;
+                        }
+                    } catch (DeviceConfigNotFoundException e) {
+                        log.warn(e.getMessage() + "aborting redoRouting");
+                        return false;
+                    }
+                    // route is from another edge to this edge-pair
+                    if (targetRoutes.containsKey(targetSw)) {
+                        targetRoutes.get(targetSw).add(route);
+                    } else {
+                        Set<ArrayList<DeviceId>> temp = new HashSet<>();
+                        temp.add(route);
+                        targetRoutes.put(targetSw, temp);
+                    }
+                    i.remove();
+                }
+            }
+            // so now for this edgepair we have a per target set of routechanges
+            // process target->edgePair route
+            for (Map.Entry<DeviceId, Set<ArrayList<DeviceId>>> entry :
+                            targetRoutes.entrySet()) {
+                log.debug("* redoRoutingDstPair Target:{} -> edge-pair {}",
+                          entry.getKey(), ep);
+                DeviceId targetSw = entry.getKey();
+                Map<DeviceId, Set<DeviceId>> perDstNextHops = new HashMap<>();
+                entry.getValue().forEach(route -> {
+                    Set<DeviceId> nhops = getNextHops(route.get(0), route.get(1));
+                    log.debug("route: target {} -> dst {} found with next-hops {}",
+                              route.get(0), route.get(1), nhops);
+                    perDstNextHops.put(route.get(1), nhops);
+                });
+                Set<IpPrefix> ipDev1 = (subnets == null) ? config.getSubnets(ep.dev1)
+                                                         : subnets;
+                Set<IpPrefix> ipDev2 = (subnets == null) ? config.getSubnets(ep.dev2)
+                                                         : subnets;
+                ipDev1 = (ipDev1 == null) ? Sets.newHashSet() : ipDev1;
+                ipDev2 = (ipDev2 == null) ? Sets.newHashSet() : ipDev2;
+                // handle routing to subnets common to edge-pair
+                // only if the targetSw is not part of the edge-pair
+                if (!ep.includes(targetSw)) {
+                    if (!populateEcmpRoutingRulePartial(
+                             targetSw,
+                             ep.dev1, ep.dev2,
+                             perDstNextHops,
+                             Sets.intersection(ipDev1, ipDev2))) {
+                        return false; // abort everything and fail fast
+                    }
+                }
+                // handle routing to subnets that only belong to dev1
+                Set<IpPrefix> onlyDev1Subnets = Sets.difference(ipDev1, ipDev2);
+                if (!onlyDev1Subnets.isEmpty() && perDstNextHops.get(ep.dev1) != null) {
+                    Map<DeviceId, Set<DeviceId>> onlyDev1NextHops = new HashMap<>();
+                    onlyDev1NextHops.put(ep.dev1, perDstNextHops.get(ep.dev1));
+                    if (!populateEcmpRoutingRulePartial(
+                            targetSw,
+                            ep.dev1, null,
+                            onlyDev1NextHops,
+                            onlyDev1Subnets)) {
+                        return false; // abort everything and fail fast
+                    }
+                }
+                // handle routing to subnets that only belong to dev2
+                Set<IpPrefix> onlyDev2Subnets = Sets.difference(ipDev2, ipDev1);
+                if (!onlyDev2Subnets.isEmpty() && perDstNextHops.get(ep.dev2) != null) {
+                    Map<DeviceId, Set<DeviceId>> onlyDev2NextHops = new HashMap<>();
+                    onlyDev2NextHops.put(ep.dev2, perDstNextHops.get(ep.dev2));
+                    if (!populateEcmpRoutingRulePartial(
+                            targetSw,
+                            ep.dev2, null,
+                            onlyDev2NextHops,
+                            onlyDev2Subnets)) {
+                        return false; // abort everything and fail fast
+                    }
+                }
+            }
+            // if it gets here it has succeeded for all targets to this edge-pair
+        }
+        return true;
+    }
+
+    /**
+     * Programs targetSw in the changedRoutes for given prefixes reachable by
+     * a destination switch that is not part of an edge-pair.
+     * If no prefixes are given, the method will use configured subnets/prefixes.
+     *
+     * @param subnets a set of prefixes that need to be populated in the routing
+     *                 table of the target switch in the changedRoutes. Can be null,
+     *                 in which case all the configured prefixes belonging to the
+     *                 paired switches will be populated in the target switch
+     * @param changedRoutes a set of route-path changes, where each route-path is
+     *                     a list with its first element the src-switch (target)
+     *                     of the path, and the second element the dst-switch of
+     *                     the path.
+     * @return true if successful
+     */
+    private boolean redoRoutingIndividualDests(Set<IpPrefix> subnets,
+                                               Set<ArrayList<DeviceId>> changedRoutes) {
+        // aggregate route-path changes for each dst device
+        HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>> routesBydevice =
+                new HashMap<>();
+        for (ArrayList<DeviceId> route: changedRoutes) {
+            DeviceId dstSw = route.get(1);
+            ArrayList<ArrayList<DeviceId>> deviceRoutes =
+                    routesBydevice.get(dstSw);
+            if (deviceRoutes == null) {
+                deviceRoutes = new ArrayList<>();
+                routesBydevice.put(dstSw, deviceRoutes);
+            }
+            deviceRoutes.add(route);
+        }
+        for (DeviceId impactedDstDevice : routesBydevice.keySet()) {
+            ArrayList<ArrayList<DeviceId>> deviceRoutes =
+                    routesBydevice.get(impactedDstDevice);
+            for (ArrayList<DeviceId> route: deviceRoutes) {
+                log.debug("* redoRoutingIndiDst Target: {} -> dst: {}",
+                          route.get(0), route.get(1));
+                DeviceId targetSw = route.get(0);
+                DeviceId dstSw = route.get(1); // same as impactedDstDevice
+                Set<DeviceId> nextHops = getNextHops(targetSw, dstSw);
+                Map<DeviceId, Set<DeviceId>> nhops = new HashMap<>();
+                nhops.put(dstSw, nextHops);
+                if (!populateEcmpRoutingRulePartial(targetSw, dstSw, null, nhops,
+                         (subnets == null) ? Sets.newHashSet() : subnets)) {
+                    return false; // abort routing and fail fast
+                }
+                log.debug("Populating flow rules from target: {} to dst: {}"
+                        + " is successful", targetSw, dstSw);
+            }
+            //Only if all the flows for all impacted routes to a
+            //specific target are pushed successfully, update the
+            //ECMP graph for that target. Or else the next event
+            //would not see any changes in the ECMP graphs.
+            //In another case, the target switch has gone away, so
+            //routes can't be installed. In that case, the current map
+            //is updated here, without any flows being pushed.
+            currentEcmpSpgMap.put(impactedDstDevice,
+                                  updatedEcmpSpgMap.get(impactedDstDevice));
+            log.debug("Updating ECMPspg for impacted dev:{}", impactedDstDevice);
+        }
+        return true;
+    }
+
+    /**
+     * Populate ECMP rules for subnets from target to destination via nexthops.
+     *
+     * @param targetSw Device ID of target switch in which rules will be programmed
+     * @param destSw1 Device ID of final destination switch to which the rules will forward
+     * @param destSw2 Device ID of paired destination switch to which the rules will forward
+     *                A null deviceId indicates packets should only be sent to destSw1
+     * @param nextHops Map indication a list of next hops per destSw
+     * @param subnets Subnets to be populated. If empty, populate all configured subnets.
+     * @return true if it succeeds in populating rules
+     */ // refactor
+    private boolean populateEcmpRoutingRulePartial(DeviceId targetSw,
+                                                   DeviceId destSw1,
+                                                   DeviceId destSw2,
+                                                   Map<DeviceId, Set<DeviceId>> nextHops,
+                                                   Set<IpPrefix> subnets) {
+        boolean result;
+        // If both target switch and dest switch are edge routers, then set IP
+        // rule for both subnet and router IP.
+        boolean targetIsEdge;
+        boolean dest1IsEdge;
+        Ip4Address dest1RouterIpv4, dest2RouterIpv4 = null;
+        Ip6Address dest1RouterIpv6, dest2RouterIpv6 = null;
+
+        try {
+            targetIsEdge = config.isEdgeDevice(targetSw);
+            dest1IsEdge = config.isEdgeDevice(destSw1);
+            dest1RouterIpv4 = config.getRouterIpv4(destSw1);
+            dest1RouterIpv6 = config.getRouterIpv6(destSw1);
+            if (destSw2 != null) {
+                dest2RouterIpv4 = config.getRouterIpv4(destSw2);
+                dest2RouterIpv6 = config.getRouterIpv6(destSw2);
+            }
+        } catch (DeviceConfigNotFoundException e) {
+            log.warn(e.getMessage() + " Aborting populateEcmpRoutingRulePartial.");
             return false;
         }
-        Boolean result = srManager.isSeenLinkUp(reverseLink);
-        if (result == null) {
-            return false;
+
+        if (targetIsEdge && dest1IsEdge) {
+            subnets = (subnets != null && !subnets.isEmpty())
+                            ? Sets.newHashSet(subnets)
+                            : Sets.newHashSet(config.getSubnets(destSw1));
+            // XXX -  Rethink this
+            /*subnets.add(dest1RouterIpv4.toIpPrefix());
+            if (dest1RouterIpv6 != null) {
+                subnets.add(dest1RouterIpv6.toIpPrefix());
+            }
+            if (destSw2 != null && dest2RouterIpv4 != null) {
+                subnets.add(dest2RouterIpv4.toIpPrefix());
+                if (dest2RouterIpv6 != null) {
+                    subnets.add(dest2RouterIpv6.toIpPrefix());
+                }
+            }*/
+            log.debug(". populateEcmpRoutingRulePartial in device {} towards {} {} "
+                    + "for subnets {}", targetSw, destSw1,
+                                        (destSw2 != null) ? ("& " + destSw2) : "",
+                                        subnets);
+            result = rulePopulator.populateIpRuleForSubnet(targetSw, subnets,
+                                                           destSw1, destSw2,
+                                                           nextHops);
+            if (!result) {
+                return false;
+            }
+            /* XXX rethink this
+            IpPrefix routerIpPrefix = destRouterIpv4.toIpPrefix();
+            log.debug("* populateEcmpRoutingRulePartial in device {} towards {} "
+                    + "for router IP {}", targetSw, destSw, routerIpPrefix);
+            result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix,
+                                                           destSw, nextHops);
+            if (!result) {
+                return false;
+            }
+            // If present we deal with IPv6 loopback.
+            if (destRouterIpv6 != null) {
+                routerIpPrefix = destRouterIpv6.toIpPrefix();
+                log.debug("* populateEcmpRoutingRulePartial in device {} towards {}"
+                        + " for v6 router IP {}", targetSw, destSw, routerIpPrefix);
+                result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix,
+                                                               destSw, nextHops);
+                if (!result) {
+                    return false;
+                }
+            }*/
         }
-        return result.booleanValue();
+
+        if (!targetIsEdge && dest1IsEdge) {
+            // MPLS rules in all non-edge target devices. These rules are for
+            // individual destinations, even if the dsts are part of edge-pairs.
+            log.debug(". populateEcmpRoutingRulePartial in device{} towards {} for "
+                    + "all MPLS rules", targetSw, destSw1);
+            result = rulePopulator.populateMplsRule(targetSw, destSw1,
+                                                    nextHops.get(destSw1),
+                                                    dest1RouterIpv4);
+            if (!result) {
+                return false;
+            }
+            if (dest1RouterIpv6 != null) {
+                result = rulePopulator.populateMplsRule(targetSw, destSw1,
+                                                        nextHops.get(destSw1),
+                                                        dest1RouterIpv6);
+                if (!result) {
+                    return false;
+                }
+            }
+        }
+
+        // To save on ECMP groups
+        // avoid MPLS rules in non-edge-devices to non-edge-devices
+        // avoid MPLS transit rules in edge-devices
+        // avoid loopback IP rules in edge-devices to non-edge-devices
+        return true;
     }
 
     /**
@@ -338,7 +821,7 @@
             }
 
             if (linkOrSwitchFailed) {
-                success = fixHashGroupsForRoute(route, true);
+                fixHashGroupsForRoute(route, true);
                 // it's possible that we cannot fix hash groups for a route
                 // if the target switch has failed. Nevertheless the ecmp graph
                 // for the impacted switch must still be updated.
@@ -352,8 +835,9 @@
                 }
                 //linkfailed - update both sides
                 currentEcmpSpgMap.put(targetSw, updatedEcmpSpgMap.get(targetSw));
-                dstSw = route.get(1);
-                currentEcmpSpgMap.put(dstSw, updatedEcmpSpgMap.get(dstSw));
+                if (dstSw != null) {
+                    currentEcmpSpgMap.put(dstSw, updatedEcmpSpgMap.get(dstSw));
+                }
                 log.debug("Updating ECMPspg for dst:{} and target:{}", dstSw, targetSw);
             } else {
                 success = fixHashGroupsForRoute(route, false);
@@ -419,6 +903,7 @@
                     }
                     for (ArrayList<DeviceId> via : swViaMap.get(target)) {
                         if (via.isEmpty()) {
+                            // the dstSw is the next-hop from the targetSw
                             nextHops.add(destSw);
                         } else {
                             // first elem is next-hop in each ECMP path
@@ -448,97 +933,59 @@
     }
 
     /**
-     * Processes a set a route-path changes by reprogramming routing rules and
-     * creating new hash-groups if necessary.
-     *
-     * @param routeChanges a set of route-path changes, where each route-path is
-     *                     a list with its first element the src-switch of the path
-     *                     and the second element the dst-switch of the path.
-     * @return true if successful in repopulating routes
+     * Start the flow rule population process if it was never started. The
+     * process finishes successfully when all flow rules are set and stops with
+     * ABORTED status when any groups required for flows is not set yet.
      */
-    private boolean repopulateRoutingRulesForRoutes(Set<ArrayList<DeviceId>> routeChanges) {
-        rulePopulator.resetCounter();
-        HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>> routesBydevice =
-                new HashMap<>();
-        for (ArrayList<DeviceId> link: routeChanges) {
-            // When only the source device is defined, reinstall routes to all other devices
-            if (link.size() == 1) {
-                log.debug("-- repopulateRoutingRulesForRoutes: running ECMP graph for device {}", link.get(0));
-                EcmpShortestPathGraph ecmpSpg = new EcmpShortestPathGraph(link.get(0), srManager);
-                if (populateEcmpRoutingRules(link.get(0), ecmpSpg, ImmutableSet.of())) {
-                    log.debug("Populating flow rules from all to dest:{} is successful",
-                              link.get(0));
-                    currentEcmpSpgMap.put(link.get(0), ecmpSpg);
-                    log.debug("Updating ECMPspg for dest:{}", link.get(0));
-                } else {
-                    log.warn("Failed to populate the flow rules from all to dest:{}", link.get(0));
-                    return false;
-                }
+    public void startPopulationProcess() {
+        statusLock.lock();
+        try {
+            if (populationStatus == Status.IDLE
+                    || populationStatus == Status.SUCCEEDED
+                    || populationStatus == Status.ABORTED) {
+                populateAllRoutingRules();
             } else {
-                ArrayList<ArrayList<DeviceId>> deviceRoutes =
-                        routesBydevice.get(link.get(1));
-                if (deviceRoutes == null) {
-                    deviceRoutes = new ArrayList<>();
-                    routesBydevice.put(link.get(1), deviceRoutes);
-                }
-                deviceRoutes.add(link);
+                log.warn("Not initiating startPopulationProcess as populationStatus is {}",
+                         populationStatus);
             }
+        } finally {
+            statusLock.unlock();
         }
-
-        for (DeviceId impactedDevice : routesBydevice.keySet()) {
-            ArrayList<ArrayList<DeviceId>> deviceRoutes =
-                    routesBydevice.get(impactedDevice);
-            for (ArrayList<DeviceId> link: deviceRoutes) {
-                log.debug("-- repopulateRoutingRulesForRoutes {} -> {}",
-                          link.get(0), link.get(1));
-                DeviceId src = link.get(0);
-                DeviceId dst = link.get(1);
-                EcmpShortestPathGraph ecmpSpg = updatedEcmpSpgMap.get(dst);
-                HashMap<Integer, HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>>> switchVia =
-                        ecmpSpg.getAllLearnedSwitchesAndVia();
-                for (Integer itrIdx : switchVia.keySet()) {
-                    HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>> swViaMap =
-                            switchVia.get(itrIdx);
-                    for (DeviceId targetSw : swViaMap.keySet()) {
-                        if (!targetSw.equals(src)) {
-                            continue;
-                        }
-                        Set<DeviceId> nextHops = new HashSet<>();
-                        for (ArrayList<DeviceId> via : swViaMap.get(targetSw)) {
-                            // in this ECMP path to the targetSw, get the next hop
-                            if (via.isEmpty()) {
-                                nextHops.add(dst);
-                            } else {
-                                nextHops.add(via.get(0));
-                            }
-                        }
-                        if (!populateEcmpRoutingRulePartial(targetSw, dst,
-                                nextHops, ImmutableSet.of())) {
-                            return false;
-                        }
-                        log.debug("Populating flow rules from {} to {} is successful",
-                                  targetSw, dst);
-                    }
-                }
-            }
-            //Only if all the flows for all impacted routes to a
-            //specific target are pushed successfully, update the
-            //ECMP graph for that target. Or else the next event
-            //would not see any changes in the ECMP graphs.
-            //In another case, the target switch has gone away, so
-            //routes can't be installed. In that case, the current map
-            //is updated here, without any flows being pushed.
-            currentEcmpSpgMap.put(impactedDevice,
-                                  updatedEcmpSpgMap.get(impactedDevice));
-            log.debug("Updating ECMPspg for impacted dev:{}", impactedDevice);
-        }
-
-        processHashGroupChange(routeChanges, false, null);
-
-        return true;
     }
 
     /**
+     * Revoke rules of given subnet in all edge switches.
+     *
+     * @param subnets subnet being removed
+     * @return true if succeed
+     */
+    protected boolean revokeSubnet(Set<IpPrefix> subnets) {
+        statusLock.lock();
+        try {
+            return srManager.routingRulePopulator.revokeIpRuleForSubnet(subnets);
+        } finally {
+            statusLock.unlock();
+        }
+    }
+
+    /**
+     * Remove ECMP graph entry for the given device. Typically called when
+     * device is no longer available.
+     *
+     * @param deviceId the device for which graphs need to be purged
+     */
+    protected void purgeEcmpGraph(DeviceId deviceId) {
+        currentEcmpSpgMap.remove(deviceId);
+        if (updatedEcmpSpgMap != null) {
+            updatedEcmpSpgMap.remove(deviceId);
+        }
+    }
+
+    //////////////////////////////////////
+    //  Routing helper methods and classes
+    //////////////////////////////////////
+
+    /**
      * Computes set of affected routes due to failed link. Assumes
      * previous ecmp shortest-path graph exists for a switch in order to compute
      * affected routes. If such a graph does not exist, the method returns null.
@@ -616,46 +1063,47 @@
      *         affected
      */
     private Set<ArrayList<DeviceId>> computeRouteChange() {
-
-        ImmutableSet.Builder<ArrayList<DeviceId>> changedRoutesBuilder =
+        ImmutableSet.Builder<ArrayList<DeviceId>> changedRtBldr =
                 ImmutableSet.builder();
 
         for (Device sw : srManager.deviceService.getDevices()) {
-            DeviceId rootSw = sw.id();
-            log.debug("Computing the impacted routes for device {}", rootSw);
-            if (!srManager.mastershipService.isLocalMaster(rootSw)) {
-                log.debug("No mastership for {} ... skipping route optimization",
-                          rootSw);
+            log.debug("Computing the impacted routes for device {}", sw.id());
+            DeviceId retId = shouldHandleRouting(sw.id());
+            if (retId == null) {
                 continue;
             }
-            if (log.isTraceEnabled()) {
-                log.trace("Device links for dev: {}", rootSw);
-                for (Link link: srManager.linkService.getDeviceLinks(rootSw)) {
-                    log.trace("{} -> {} ", link.src().deviceId(), link.dst().deviceId());
+            Set<DeviceId> devicesToProcess = Sets.newHashSet(retId, sw.id());
+            for (DeviceId rootSw : devicesToProcess) {
+                if (log.isTraceEnabled()) {
+                    log.trace("Device links for dev: {}", rootSw);
+                    for (Link link: srManager.linkService.getDeviceLinks(rootSw)) {
+                        log.trace("{} -> {} ", link.src().deviceId(),
+                                  link.dst().deviceId());
+                    }
                 }
+                EcmpShortestPathGraph currEcmpSpg = currentEcmpSpgMap.get(rootSw);
+                if (currEcmpSpg == null) {
+                    log.debug("No existing ECMP graph for device {}.. adding self as "
+                            + "changed route", rootSw);
+                    changedRtBldr.add(Lists.newArrayList(rootSw));
+                    continue;
+                }
+                EcmpShortestPathGraph newEcmpSpg = updatedEcmpSpgMap.get(rootSw);
+                if (log.isDebugEnabled()) {
+                    log.debug("Root switch: {}", rootSw);
+                    log.debug("  Current/Existing SPG: {}", currEcmpSpg);
+                    log.debug("       New/Updated SPG: {}", newEcmpSpg);
+                }
+                // first use the updated/new map to compare to current/existing map
+                // as new links may have come up
+                changedRtBldr.addAll(compareGraphs(newEcmpSpg, currEcmpSpg, rootSw));
+                // then use the current/existing map to compare to updated/new map
+                // as switch may have been removed
+                changedRtBldr.addAll(compareGraphs(currEcmpSpg, newEcmpSpg, rootSw));
             }
-            EcmpShortestPathGraph currEcmpSpg = currentEcmpSpgMap.get(rootSw);
-            if (currEcmpSpg == null) {
-                log.debug("No existing ECMP graph for device {}.. adding self as "
-                        + "changed route", rootSw);
-                changedRoutesBuilder.add(Lists.newArrayList(rootSw));
-                continue;
-            }
-            EcmpShortestPathGraph newEcmpSpg = updatedEcmpSpgMap.get(rootSw);
-            if (log.isDebugEnabled()) {
-                log.debug("Root switch: {}", rootSw);
-                log.debug("  Current/Existing SPG: {}", currEcmpSpg);
-                log.debug("       New/Updated SPG: {}", newEcmpSpg);
-            }
-            // first use the updated/new map to compare to current/existing map
-            // as new links may have come up
-            changedRoutesBuilder.addAll(compareGraphs(newEcmpSpg, currEcmpSpg, rootSw));
-            // then use the current/existing map to compare to updated/new map
-            // as switch may have been removed
-            changedRoutesBuilder.addAll(compareGraphs(currEcmpSpg, newEcmpSpg, rootSw));
         }
 
-        Set<ArrayList<DeviceId>> changedRoutes = changedRoutesBuilder.build();
+        Set<ArrayList<DeviceId>> changedRoutes = changedRtBldr.build();
         for (ArrayList<DeviceId> route: changedRoutes) {
             log.debug("Route changes Target -> Root");
             if (route.size() == 1) {
@@ -695,8 +1143,8 @@
                 if ((compPath == null) || !basePath.equals(compPath)) {
                     log.trace("Impacted route:{} -> {}", targetSw, rootSw);
                     ArrayList<DeviceId> route = new ArrayList<>();
-                    route.add(targetSw);
-                    route.add(rootSw);
+                    route.add(targetSw); // switch with rules to populate
+                    route.add(rootSw); // towards this destination
                     changedRoutesBuilder.add(route);
                 }
             }
@@ -704,6 +1152,13 @@
         return changedRoutesBuilder.build();
     }
 
+    /**
+     * Returns the ECMP paths traversed to reach the target switch.
+     *
+     * @param switchVia a per-iteration view of the ECMP graph for a root switch
+     * @param targetSw the switch to reach from the root switch
+     * @return the nodes traversed on ECMP paths to the target switch
+     */
     private ArrayList<ArrayList<DeviceId>> getVia(HashMap<Integer, HashMap<DeviceId,
             ArrayList<ArrayList<DeviceId>>>> switchVia, DeviceId targetSw) {
         for (Integer itrIdx : switchVia.keySet()) {
@@ -719,6 +1174,15 @@
         return null;
     }
 
+    /**
+     * Utility method to break down a path from src to dst device into a collection
+     * of links.
+     *
+     * @param src src device of the path
+     * @param dst dst device of the path
+     * @param viaMap path taken from src to dst device
+     * @return collection of links in the path
+     */
     private Set<ArrayList<DeviceId>> computeLinks(DeviceId src,
                                                   DeviceId dst,
                        HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>> viaMap) {
@@ -744,133 +1208,175 @@
     }
 
     /**
-     * Populate ECMP rules for subnets from all switches to destination.
+     * Determines whether this controller instance should handle routing for the
+     * given {@code deviceId}, based on mastership and pairDeviceId if one exists.
+     * Returns null if this instance should not handle routing for given {@code deviceId}.
+     * Otherwise the returned value could be the given deviceId itself, or the
+     * deviceId for the paired edge device. In the latter case, this instance
+     * should handle routing for both the given device and the paired device.
      *
-     * @param destSw Device ID of destination switch
-     * @param ecmpSPG ECMP shortest path graph
-     * @param subnets Subnets to be populated. If empty, populate all configured subnets.
-     * @return true if it succeeds in populating rules
+     * @param deviceId device identifier to consider for routing
+     * @return null or deviceId which could be the same as the given deviceId
+     *          or the deviceId of a paired edge device
      */
-    private boolean populateEcmpRoutingRules(DeviceId destSw,
-                                             EcmpShortestPathGraph ecmpSPG,
-                                             Set<IpPrefix> subnets) {
+    private DeviceId shouldHandleRouting(DeviceId deviceId) {
+        if (!srManager.mastershipService.isLocalMaster(deviceId)) {
+            log.debug("Not master for dev:{} .. skipping routing, may get handled "
+                    + "elsewhere as part of paired devices", deviceId);
+            return null;
+        }
+        NodeId myNode = srManager.mastershipService.getMasterFor(deviceId);
+        DeviceId pairDev = getPairDev(deviceId);
 
-        HashMap<Integer, HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>>> switchVia = ecmpSPG
-                .getAllLearnedSwitchesAndVia();
-        for (Integer itrIdx : switchVia.keySet()) {
-            HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>> swViaMap = switchVia
-                    .get(itrIdx);
-            for (DeviceId targetSw : swViaMap.keySet()) {
-                Set<DeviceId> nextHops = new HashSet<>();
-                log.debug("** Iter: {} root: {} target: {}", itrIdx, destSw, targetSw);
-                for (ArrayList<DeviceId> via : swViaMap.get(targetSw)) {
-                    if (via.isEmpty()) {
-                        nextHops.add(destSw);
-                    } else {
-                        nextHops.add(via.get(0));
-                    }
-                }
-                if (!populateEcmpRoutingRulePartial(targetSw, destSw, nextHops, subnets)) {
-                    return false;
-                }
+        if (pairDev != null) {
+            if (!srManager.deviceService.isAvailable(pairDev)) {
+                log.warn("pairedDev {} not available .. routing this dev:{} "
+                        + "without mastership check",
+                          pairDev, deviceId);
+                return pairDev; // handle both temporarily
+            }
+            NodeId pairMasterNode = srManager.mastershipService.getMasterFor(pairDev);
+            if (myNode.compareTo(pairMasterNode) <= 0) {
+                log.debug("Handling routing for both dev:{} pair-dev:{}; myNode: {}"
+                        + " pairMaster:{} compare:{}", deviceId, pairDev,
+                        myNode, pairMasterNode,
+                        myNode.compareTo(pairMasterNode));
+                return pairDev; // handle both
+            } else {
+                log.debug("PairDev node: {} should handle routing for dev:{} and "
+                        + "pair-dev:{}", pairMasterNode, deviceId, pairDev);
+                return null; // handle neither
             }
         }
-
-        return true;
+        return deviceId; // not paired, just handle given device
     }
 
     /**
-     * Populate ECMP rules for subnets from target to destination via nexthops.
+     * Returns the configured paired DeviceId for the given Device, or null
+     * if no such paired device has been configured.
      *
-     * @param targetSw Device ID of target switch in which rules will be programmed
-     * @param destSw Device ID of final destination switch to which the rules will forward
-     * @param nextHops List of next hops via which destSw will be reached
-     * @param subnets Subnets to be populated. If empty, populate all configured subnets.
-     * @return true if it succees in populating rules
+     * @param deviceId
+     * @return configured pair deviceId or null
      */
-    private boolean populateEcmpRoutingRulePartial(DeviceId targetSw,
-                                                   DeviceId destSw,
-                                                   Set<DeviceId> nextHops,
-                                                   Set<IpPrefix> subnets) {
-        boolean result;
-
-        if (nextHops.isEmpty()) {
-            nextHops.add(destSw);
-        }
-        // If both target switch and dest switch are edge routers, then set IP
-        // rule for both subnet and router IP.
-        boolean targetIsEdge;
-        boolean destIsEdge;
-        Ip4Address destRouterIpv4;
-        Ip6Address destRouterIpv6;
-
+    private DeviceId getPairDev(DeviceId deviceId) {
+        DeviceId pairDev;
         try {
-            targetIsEdge = config.isEdgeDevice(targetSw);
-            destIsEdge = config.isEdgeDevice(destSw);
-            destRouterIpv4 = config.getRouterIpv4(destSw);
-            destRouterIpv6 = config.getRouterIpv6(destSw);
+            pairDev = srManager.deviceConfiguration.getPairDeviceId(deviceId);
         } catch (DeviceConfigNotFoundException e) {
-            log.warn(e.getMessage() + " Aborting populateEcmpRoutingRulePartial.");
-            return false;
+            log.warn(e.getMessage() + " .. cannot continue routing for dev: {}");
+            return null;
         }
-
-        if (targetIsEdge && destIsEdge) {
-            subnets = (subnets != null && !subnets.isEmpty()) ? subnets
-                                                              : config.getSubnets(destSw);
-            log.debug("* populateEcmpRoutingRulePartial in device {} towards {} "
-                    + "for subnets {}", targetSw, destSw, subnets);
-            result = rulePopulator.populateIpRuleForSubnet(targetSw, subnets,
-                                                           destSw, nextHops);
-            if (!result) {
-                return false;
-            }
-            IpPrefix routerIpPrefix = destRouterIpv4.toIpPrefix();
-            log.debug("* populateEcmpRoutingRulePartial in device {} towards {} "
-                    + "for router IP {}", targetSw, destSw, routerIpPrefix);
-            result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix,
-                                                           destSw, nextHops);
-            if (!result) {
-                return false;
-            }
-            // If present we deal with IPv6 loopback.
-            if (destRouterIpv6 != null) {
-                routerIpPrefix = destRouterIpv6.toIpPrefix();
-                log.debug("* populateEcmpRoutingRulePartial in device {} towards {}"
-                        + " for v6 router IP {}", targetSw, destSw, routerIpPrefix);
-                result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix,
-                                                               destSw, nextHops);
-                if (!result) {
-                    return false;
-                }
-            }
-        }
-
-        if (!targetIsEdge && destIsEdge) {
-            // MPLS rules in all non-edge target devices
-            log.debug("* populateEcmpRoutingRulePartial in device{} towards {} for "
-                    + "all MPLS rules", targetSw, destSw);
-            result = rulePopulator.populateMplsRule(targetSw, destSw, nextHops,
-                                                    destRouterIpv4);
-            if (!result) {
-                return false;
-            }
-            if (destRouterIpv6 != null) {
-                result = rulePopulator.populateMplsRule(targetSw, destSw, nextHops,
-                                                        destRouterIpv6);
-                if (!result) {
-                    return false;
-                }
-            }
-        }
-
-        // To save on ECMP groups
-        // avoid MPLS rules in non-edge-devices to non-edge-devices
-        // avoid MPLS transit rules in edge-devices
-        // avoid loopback IP rules in edge-devices to non-edge-devices
-        return true;
+        return pairDev;
     }
 
     /**
+     * Returns the set of deviceIds which are the next hops from the targetSw
+     * to the dstSw according to the latest ECMP spg.
+     *
+     * @param targetSw the switch for which the next-hops are desired
+     * @param dstSw the switch to which the next-hops lead to from the targetSw
+     * @return set of next hop deviceIds, could be empty if no next hops are found
+     */
+    private Set<DeviceId> getNextHops(DeviceId targetSw, DeviceId dstSw) {
+        boolean targetIsEdge = false;
+        try {
+            targetIsEdge = srManager.deviceConfiguration.isEdgeDevice(targetSw);
+        } catch (DeviceConfigNotFoundException e) {
+            log.warn(e.getMessage() + "Cannot determine if targetIsEdge {}.. "
+                    + "continuing to getNextHops", targetSw);
+        }
+
+        EcmpShortestPathGraph ecmpSpg = updatedEcmpSpgMap.get(dstSw);
+        if (ecmpSpg == null) {
+            log.debug("No ecmpSpg found for dstSw: {}", dstSw);
+            return ImmutableSet.of();
+        }
+        HashMap<Integer,
+            HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>>> switchVia =
+                ecmpSpg.getAllLearnedSwitchesAndVia();
+        for (Integer itrIdx : switchVia.keySet()) {
+            HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>> swViaMap =
+                    switchVia.get(itrIdx);
+            for (DeviceId target : swViaMap.keySet()) {
+                if (!target.equals(targetSw)) {
+                    continue;
+                }
+                if (!targetIsEdge && itrIdx > 1) {
+                    // optimization for spines to not use other leaves to get
+                    // to a leaf to avoid loops
+                    log.debug("Avoiding {} hop path for non-edge targetSw:{}"
+                            + " --> dstSw:{}", itrIdx, targetSw, dstSw);
+                    break;
+                }
+                Set<DeviceId> nextHops = new HashSet<>();
+                for (ArrayList<DeviceId> via : swViaMap.get(targetSw)) {
+                    if (via.isEmpty()) {
+                        // the dstSw is the next-hop from the targetSw
+                        nextHops.add(dstSw);
+                    } else {
+                        // first elem is next-hop in each ECMP path
+                        nextHops.add(via.get(0));
+                    }
+                }
+                return nextHops;
+            }
+        }
+        return ImmutableSet.of(); //no next-hops found
+    }
+
+    /**
+     * Represents two devices that are paired by configuration. An EdgePair for
+     * (dev1, dev2) is the same as as EdgePair for (dev2, dev1)
+     */
+    protected final class EdgePair {
+        DeviceId dev1;
+        DeviceId dev2;
+
+        EdgePair(DeviceId dev1, DeviceId dev2) {
+            this.dev1 = dev1;
+            this.dev2 = dev2;
+        }
+
+        boolean includes(DeviceId dev) {
+            return dev1.equals(dev) || dev2.equals(dev);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (!(o instanceof EdgePair)) {
+                return false;
+            }
+            EdgePair that = (EdgePair) o;
+            return ((this.dev1.equals(that.dev1) && this.dev2.equals(that.dev2)) ||
+                    (this.dev1.equals(that.dev2) && this.dev2.equals(that.dev1)));
+        }
+
+        @Override
+        public int hashCode() {
+            if (dev1.toString().compareTo(dev2.toString()) <= 0) {
+                return Objects.hash(dev1, dev2);
+            } else {
+                return Objects.hash(dev2, dev1);
+            }
+        }
+
+        @Override
+        public String toString() {
+            return toStringHelper(this)
+                    .add("Dev1", dev1)
+                    .add("Dev2", dev2)
+                    .toString();
+        }
+    }
+
+    //////////////////////////////////////
+    //  Filtering rule creation
+    //////////////////////////////////////
+
+    /**
      * Populates filtering rules for port, and punting rules
      * for gateway IPs, loopback IPs and arp/ndp traffic.
      * Should only be called by the master instance for this device/port.
@@ -891,91 +1397,6 @@
     }
 
     /**
-     * Start the flow rule population process if it was never started. The
-     * process finishes successfully when all flow rules are set and stops with
-     * ABORTED status when any groups required for flows is not set yet.
-     */
-    public void startPopulationProcess() {
-        statusLock.lock();
-        try {
-            if (populationStatus == Status.IDLE
-                    || populationStatus == Status.SUCCEEDED
-                    || populationStatus == Status.ABORTED) {
-                populationStatus = Status.STARTED;
-                populateAllRoutingRules();
-            } else {
-                log.warn("Not initiating startPopulationProcess as populationStatus is {}",
-                         populationStatus);
-            }
-        } finally {
-            statusLock.unlock();
-        }
-    }
-
-    /**
-     * Resume the flow rule population process if it was aborted for any reason.
-     * Mostly the process is aborted when the groups required are not set yet.
-     *  XXX is this called?
-     *
-     */
-    public void resumePopulationProcess() {
-        statusLock.lock();
-        try {
-            if (populationStatus == Status.ABORTED) {
-                populationStatus = Status.STARTED;
-                // TODO: we need to restart from the point aborted instead of
-                // restarting.
-                populateAllRoutingRules();
-            }
-        } finally {
-            statusLock.unlock();
-        }
-    }
-
-    /**
-     * Populate rules of given subnet at given location.
-     *
-     * @param cp connect point of the subnet being added
-     * @param subnets subnet being added
-     * @return true if succeed
-     */
-    protected boolean populateSubnet(ConnectPoint cp, Set<IpPrefix> subnets) {
-        statusLock.lock();
-        try {
-            EcmpShortestPathGraph ecmpSpg = currentEcmpSpgMap.get(cp.deviceId());
-            if (ecmpSpg == null) {
-                log.warn("Fail to populating subnet {}: {}", subnets, ECMPSPG_MISSING);
-                return false;
-            }
-            return populateEcmpRoutingRules(cp.deviceId(), ecmpSpg, subnets);
-        } finally {
-            statusLock.unlock();
-        }
-    }
-
-    /**
-     * Revoke rules of given subnet at given location.
-     *
-     * @param subnets subnet being removed
-     * @return true if succeed
-     */
-    protected boolean revokeSubnet(Set<IpPrefix> subnets) {
-        statusLock.lock();
-        try {
-            return srManager.routingRulePopulator.revokeIpRuleForSubnet(subnets);
-        } finally {
-            statusLock.unlock();
-        }
-    }
-
-    protected void purgeEcmpGraph(DeviceId deviceId) {
-        currentEcmpSpgMap.remove(deviceId);
-        if (updatedEcmpSpgMap != null) {
-            updatedEcmpSpgMap.remove(deviceId);
-        }
-    }
-
-    /**
      * Utility class used to temporarily store information about the ports on a
      * device processed for filtering objectives.
      */
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/EcmpShortestPathGraph.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/EcmpShortestPathGraph.java
index 1abef8d..202a564 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/EcmpShortestPathGraph.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/EcmpShortestPathGraph.java
@@ -24,17 +24,18 @@
 import org.onosproject.net.provider.ProviderId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
-import java.util.List;
+import java.util.Set;
 
 /**
- * This class creates bandwidth constrained breadth first tree and returns paths
- * from root Device to leaf Devices (target devices) which satisfies the bandwidth condition. If
- * bandwidth parameter is not specified, the normal breadth first tree will be
- * calculated. The paths are snapshot paths at the point of the class
- * instantiation.
+ * This class creates breadth-first-search (BFS) tree for a given root device
+ * and returns paths from the root Device to leaf Devices (target devices).
+ * The paths are snapshot paths at the point of the class instantiation.
  */
 public class EcmpShortestPathGraph {
     LinkedList<DeviceId> deviceQueue = new LinkedList<>();
@@ -45,21 +46,7 @@
     HashMap<Integer, ArrayList<DeviceId>> distanceDeviceMap = new HashMap<>();
     DeviceId rootDevice;
     private SegmentRoutingManager srManager;
-    private static final Logger log = LoggerFactory
-            .getLogger(EcmpShortestPathGraph.class);
-
-    /**
-     * Constructor.
-     *
-     * @param rootDevice root of the BFS tree
-     * @param linkListToAvoid link list to avoid
-     * @param deviceIdListToAvoid device list to avoid
-     */
-    public EcmpShortestPathGraph(DeviceId rootDevice, List<String> deviceIdListToAvoid,
-                                 List<Link> linkListToAvoid) {
-        this.rootDevice = rootDevice;
-        calcECMPShortestPathGraph(deviceIdListToAvoid, linkListToAvoid);
-    }
+    private static final Logger log = LoggerFactory.getLogger(EcmpShortestPathGraph.class);
 
     /**
      * Constructor.
@@ -74,26 +61,28 @@
     }
 
     /**
-     * Calculates the BFS tree using any provided constraints and Intents.
+     * Calculates the BFS tree.
      */
-    private void calcECMPShortestPathGraph() {
+   private void calcECMPShortestPathGraph() {
         deviceQueue.add(rootDevice);
         int currDistance = 0;
         distanceQueue.add(currDistance);
         deviceSearched.put(rootDevice, currDistance);
         while (!deviceQueue.isEmpty()) {
             DeviceId sw = deviceQueue.poll();
-            DeviceId prevSw = null;
+            Set<DeviceId> prevSw = Sets.newHashSet();
             currDistance = distanceQueue.poll();
 
             for (Link link : srManager.linkService.getDeviceEgressLinks(sw)) {
+                if (srManager.avoidLink(link)) {
+                    continue;
+                }
                 DeviceId reachedDevice = link.dst().deviceId();
-                if ((prevSw != null)
-                        && (prevSw.equals(reachedDevice))) {
-                    /* Ignore LAG links between the same set of Devicees */
+                if (prevSw.contains(reachedDevice)) {
+                    // Ignore LAG links between the same set of Devices
                     continue;
                 } else  {
-                    prevSw = reachedDevice;
+                    prevSw.add(reachedDevice);
                 }
 
                 Integer distance = deviceSearched.get(reachedDevice);
@@ -101,7 +90,7 @@
                     continue;
                 }
                 if (distance == null) {
-                    /* First time visiting this Device node */
+                    // First time visiting this Device node
                     deviceQueue.add(reachedDevice);
                     distanceQueue.add(currDistance + 1);
                     deviceSearched.put(reachedDevice, currDistance + 1);
@@ -125,110 +114,13 @@
                     //upstreamLinkArray.add(link);
                     upstreamLinks.put(reachedDevice, upstreamLinkArray);
                 } else {
-                    /* ECMP links */
+                    // ECMP links
                     upstreamLinkArray.add(copyDefaultLink(link));
                 }
             }
         }
     }
 
-    /**
-     * Calculates the BFS tree using any provided constraints and Intents.
-     */
-    private void calcECMPShortestPathGraph(List<String> deviceIdListToAvoid, List<Link> linksToAvoid) {
-        deviceQueue.add(rootDevice);
-        int currDistance = 0;
-        distanceQueue.add(currDistance);
-        deviceSearched.put(rootDevice, currDistance);
-        boolean foundLinkToAvoid = false;
-        while (!deviceQueue.isEmpty()) {
-            DeviceId sw = deviceQueue.poll();
-            DeviceId prevSw = null;
-            currDistance = distanceQueue.poll();
-            for (Link link : srManager.linkService.getDeviceEgressLinks(sw)) {
-                for (Link linkToAvoid: linksToAvoid) {
-                    // TODO: equls should work
-                    //if (link.equals(linkToAvoid)) {
-                    if (linkContains(link, linksToAvoid)) {
-                        foundLinkToAvoid = true;
-                        break;
-                    }
-                }
-                if (foundLinkToAvoid) {
-                    foundLinkToAvoid = false;
-                    continue;
-                }
-                DeviceId reachedDevice = link.dst().deviceId();
-                if (deviceIdListToAvoid.contains(reachedDevice.toString())) {
-                    continue;
-                }
-                if ((prevSw != null)
-                        && (prevSw.equals(reachedDevice))) {
-                    /* Ignore LAG links between the same set of Devicees */
-                    continue;
-                } else {
-                    prevSw = reachedDevice;
-                }
-
-                Integer distance = deviceSearched.get(reachedDevice);
-                if ((distance != null) && (distance < (currDistance + 1))) {
-                    continue;
-                }
-                if (distance == null) {
-                    /* First time visiting this Device node */
-                    deviceQueue.add(reachedDevice);
-                    distanceQueue.add(currDistance + 1);
-                    deviceSearched.put(reachedDevice, currDistance + 1);
-
-                    ArrayList<DeviceId> distanceSwArray = distanceDeviceMap
-                            .get(currDistance + 1);
-                    if (distanceSwArray == null) {
-                        distanceSwArray = new ArrayList<>();
-                        distanceSwArray.add(reachedDevice);
-                        distanceDeviceMap.put(currDistance + 1, distanceSwArray);
-                    } else {
-                        distanceSwArray.add(reachedDevice);
-                    }
-                }
-
-                ArrayList<Link> upstreamLinkArray =
-                        upstreamLinks.get(reachedDevice);
-                if (upstreamLinkArray == null) {
-                    upstreamLinkArray = new ArrayList<>();
-                    upstreamLinkArray.add(copyDefaultLink(link));
-                    upstreamLinks.put(reachedDevice, upstreamLinkArray);
-                } else {
-                    /* ECMP links */
-                    upstreamLinkArray.add(copyDefaultLink(link));
-                }
-            }
-        }
-    }
-
-
-    private boolean linkContains(Link link, List<Link> links) {
-
-        DeviceId srcDevice1 = link.src().deviceId();
-        DeviceId dstDevice1 = link.dst().deviceId();
-        long srcPort1 = link.src().port().toLong();
-        long dstPort1 = link.dst().port().toLong();
-
-        for (Link link2: links) {
-            DeviceId srcDevice2 = link2.src().deviceId();
-            DeviceId dstDevice2 = link2.dst().deviceId();
-            long srcPort2 = link2.src().port().toLong();
-            long dstPort2 = link2.dst().port().toLong();
-
-            if (srcDevice1.toString().equals(srcDevice2.toString())
-                    && dstDevice1.toString().equals(dstDevice2.toString())
-                    && srcPort1 == srcPort2 && dstPort1 == dstPort2) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
     private void getDFSPaths(DeviceId dstDeviceDeviceId, Path path, ArrayList<Path> paths) {
         DeviceId rootDeviceDeviceId = rootDevice;
         for (Link upstreamLink : upstreamLinks.get(dstDeviceDeviceId)) {
@@ -360,7 +252,7 @@
     public String toString() {
         StringBuilder sBuilder = new StringBuilder();
         for (Device device: srManager.deviceService.getDevices()) {
-            if (device.id() != rootDevice) {
+            if (!device.id().equals(rootDevice)) {
                 sBuilder.append("\r\n  Paths from " + rootDevice + " to "
                                 + device.id());
                 ArrayList<Path> paths = getECMPPaths(device.id());
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/RouteHandler.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/RouteHandler.java
index 49485bb..3f7b8ad 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/RouteHandler.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/RouteHandler.java
@@ -17,6 +17,8 @@
 package org.onosproject.segmentrouting;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
@@ -68,7 +70,9 @@
         ConnectPoint location = route.location();
 
         srManager.deviceConfiguration.addSubnet(location, prefix);
-        srManager.defaultRoutingHandler.populateSubnet(location, ImmutableSet.of(prefix));
+        // XXX need to handle the case where there are two connectpoints
+        srManager.defaultRoutingHandler.populateSubnet(Sets.newHashSet(location),
+                                                       Sets.newHashSet(prefix));
         srManager.routingRulePopulator.populateRoute(location.deviceId(), prefix,
                 nextHopMac, nextHopVlan, location.port());
     }
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
index 02a0c71..9ff5bd0 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
@@ -35,7 +35,7 @@
 import org.onosproject.segmentrouting.config.DeviceConfigNotFoundException;
 import org.onosproject.segmentrouting.config.DeviceConfiguration;
 import org.onosproject.segmentrouting.grouphandler.DefaultGroupHandler;
-import org.onosproject.segmentrouting.grouphandler.NeighborSet;
+import org.onosproject.segmentrouting.grouphandler.DestinationSet;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Port;
 import org.onosproject.net.PortNumber;
@@ -56,8 +56,10 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
@@ -119,14 +121,14 @@
      */
     public void populateRoute(DeviceId deviceId, IpPrefix prefix,
                               MacAddress hostMac, VlanId hostVlanId, PortNumber outPort) {
-        log.debug("Populate routing entry for route {} at {}:{}",
+        log.debug("Populate direct routing entry for route {} at {}:{}",
                 prefix, deviceId, outPort);
         ForwardingObjective.Builder fwdBuilder;
         try {
             fwdBuilder = routingFwdObjBuilder(deviceId, prefix, hostMac,
                                               hostVlanId, outPort, false);
         } catch (DeviceConfigNotFoundException e) {
-            log.warn(e.getMessage() + " Aborting populateIpRuleForHost.");
+            log.warn(e.getMessage() + " Aborting direct populateRoute");
             return;
         }
         if (fwdBuilder == null) {
@@ -135,9 +137,10 @@
             return;
         }
         ObjectiveContext context = new DefaultObjectiveContext(
-                (objective) -> log.debug("Routing rule for route {} populated", prefix),
+                (objective) -> log.debug("Direct routing rule for route {} populated",
+                                         prefix),
                 (objective, error) ->
-                        log.warn("Failed to populate routing rule for route {}: {}",
+                        log.warn("Failed to populate direct routing rule for route {}: {}",
                                  prefix, error));
         srManager.flowObjectiveService.forward(deviceId, fwdBuilder.add(context));
         rulePopulationCounter.incrementAndGet();
@@ -258,18 +261,27 @@
     }
 
     /**
-     * Populates IP flow rules for the subnets of the destination router.
+     * Populates IP flow rules for all the given prefixes reachable from the
+     * destination switch(es).
      *
-     * @param deviceId switch ID to set the rules
-     * @param subnets subnet being added
-     * @param destSw destination switch ID
-     * @param nextHops next hop switch ID list
+     * @param targetSw switch where rules are to be programmed
+     * @param subnets subnets/prefixes being added
+     * @param destSw1 destination switch where the prefixes are reachable
+     * @param destSw2 paired destination switch if one exists for the subnets/prefixes.
+     *                Should be null if there is no paired destination switch (by config)
+     *                or if the given prefixes are reachable only via destSw1
+     * @param nextHops a map containing a set of next-hops for each destination switch.
+     *                 If destSw2 is not null, then this map must contain an
+     *                 entry for destSw2 with its next-hops from the targetSw
+     *                 (although the next-hop set may be empty in certain scenarios).
+     *                 If destSw2 is null, there should not be an entry in this
+     *                 map for destSw2.
      * @return true if all rules are set successfully, false otherwise
      */
-    public boolean populateIpRuleForSubnet(DeviceId deviceId, Set<IpPrefix> subnets,
-            DeviceId destSw, Set<DeviceId> nextHops) {
+    public boolean populateIpRuleForSubnet(DeviceId targetSw, Set<IpPrefix> subnets,
+            DeviceId destSw1, DeviceId destSw2, Map<DeviceId, Set<DeviceId>> nextHops) {
         for (IpPrefix subnet : subnets) {
-            if (!populateIpRuleForRouter(deviceId, subnet, destSw, nextHops)) {
+            if (!populateIpRuleForRouter(targetSw, subnet, destSw1, destSw2, nextHops)) {
                 return false;
             }
         }
@@ -277,7 +289,7 @@
     }
 
     /**
-     * Revokes IP flow rules for the subnets.
+     * Revokes IP flow rules for the subnets in each edge switch.
      *
      * @param subnets subnet being removed
      * @return true if all rules are removed successfully, false otherwise
@@ -293,23 +305,36 @@
 
     /**
      * Populates IP flow rules for an IP prefix in the target device. The prefix
-     * is reachable via destination device.
+     * is reachable via destination device(s).
      *
-     * @param deviceId target device ID to set the rules
-     * @param ipPrefix the destination IP prefix
-     * @param destSw device ID of the destination router
-     * @param nextHops next hop switch ID list
+     * @param targetSw target device ID to set the rules
+     * @param ipPrefix the IP prefix
+     * @param destSw1 destination switch where the prefixes are reachable
+     * @param destSw2 paired destination switch if one exists for the subnets/prefixes.
+     *                Should be null if there is no paired destination switch (by config)
+     *                or if the given prefixes are reachable only via destSw1
+     * @param nextHops map of destination switches and their next-hops.
+     *                  Should only contain destination switches that are
+     *                  actually meant to be routed to. If destSw2 is null, there
+     *                  should not be an entry for destSw2 in this map.
      * @return true if all rules are set successfully, false otherwise
      */
-    public boolean populateIpRuleForRouter(DeviceId deviceId,
-                                           IpPrefix ipPrefix, DeviceId destSw,
-                                           Set<DeviceId> nextHops) {
-        int segmentId;
+    public boolean populateIpRuleForRouter(DeviceId targetSw,
+                                           IpPrefix ipPrefix, DeviceId destSw1,
+                                           DeviceId destSw2,
+                                           Map<DeviceId, Set<DeviceId>> nextHops) {
+        int segmentId1, segmentId2 = -1;
         try {
             if (ipPrefix.isIp4()) {
-                segmentId = config.getIPv4SegmentId(destSw);
+                segmentId1 = config.getIPv4SegmentId(destSw1);
+                if (destSw2 != null) {
+                    segmentId2 = config.getIPv4SegmentId(destSw2);
+                }
             } else {
-                segmentId = config.getIPv6SegmentId(destSw);
+                segmentId1 = config.getIPv6SegmentId(destSw1);
+                if (destSw2 != null) {
+                    segmentId2 = config.getIPv6SegmentId(destSw2);
+                }
             }
         } catch (DeviceConfigNotFoundException e) {
             log.warn(e.getMessage() + " Aborting populateIpRuleForRouter.");
@@ -320,17 +345,35 @@
         TrafficSelector selector = sbuilder.build();
 
         TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
-        NeighborSet ns;
+        DestinationSet ds;
         TrafficTreatment treatment;
 
         // If the next hop is the same as the final destination, then MPLS label
         // is not set.
-        if (nextHops.size() == 1 && nextHops.toArray()[0].equals(destSw)) {
+        /*if (nextHops.size() == 1 && nextHops.toArray()[0].equals(destSw)) {
             tbuilder.immediate().decNwTtl();
-            ns = new NeighborSet(nextHops, false, destSw);
+            ds = new DestinationSet(false, destSw);
             treatment = tbuilder.build();
         } else {
-            ns = new NeighborSet(nextHops, false, segmentId, destSw);
+            ds = new DestinationSet(false, segmentId, destSw);
+            treatment = null;
+        }*/
+        if (destSw2 == null) {
+            // single dst - create destination set based on next-hop
+            Set<DeviceId> nhd1 = nextHops.get(destSw1);
+            if (nhd1.size() == 1 && nhd1.iterator().next().equals(destSw1)) {
+                tbuilder.immediate().decNwTtl();
+                ds = new DestinationSet(false, destSw1);
+                treatment = tbuilder.build();
+            } else {
+                ds = new DestinationSet(false, segmentId1, destSw1);
+                treatment = null;
+            }
+        } else {
+            // dst pair - IP rules for dst-pairs are always from other edge nodes
+            // the destination set needs to have both destinations, even if there
+            // are no next hops to one of them
+            ds = new DestinationSet(false, segmentId1, destSw1, segmentId2, destSw2);
             treatment = null;
         }
 
@@ -339,16 +382,17 @@
         // other neighboring routers, there is no subnet assigned on those ports.
         TrafficSelector.Builder metabuilder = DefaultTrafficSelector.builder(selector);
         metabuilder.matchVlanId(SegmentRoutingManager.INTERNAL_VLAN);
-        DefaultGroupHandler grpHandler = srManager.getGroupHandler(deviceId);
+        DefaultGroupHandler grpHandler = srManager.getGroupHandler(targetSw);
         if (grpHandler == null) {
             log.warn("populateIPRuleForRouter: groupHandler for device {} "
-                    + "not found", deviceId);
+                    + "not found", targetSw);
             return false;
         }
 
-        int nextId = grpHandler.getNextObjectiveId(ns, metabuilder.build(), true);
+        int nextId = grpHandler.getNextObjectiveId(ds, nextHops,
+                                                   metabuilder.build(), true);
         if (nextId <= 0) {
-            log.warn("No next objective in {} for ns: {}", deviceId, ns);
+            log.warn("No next objective in {} for ds: {}", targetSw, ds);
             return false;
         }
 
@@ -364,14 +408,14 @@
             fwdBuilder.withTreatment(treatment);
         }
         log.debug("Installing IPv4 forwarding objective for router IP/subnet {} "
-                + "in switch {} with nextId: {}", ipPrefix, deviceId, nextId);
+                + "in switch {} with nextId: {}", ipPrefix, targetSw, nextId);
         ObjectiveContext context = new DefaultObjectiveContext(
                 (objective) -> log.debug("IP rule for router {} populated in dev:{}",
-                                         ipPrefix, deviceId),
+                                         ipPrefix, targetSw),
                 (objective, error) ->
                         log.warn("Failed to populate IP rule for router {}: {} in dev:{}",
-                                 ipPrefix, error, deviceId));
-        srManager.flowObjectiveService.forward(deviceId, fwdBuilder.add(context));
+                                 ipPrefix, error, targetSw));
+        srManager.flowObjectiveService.forward(targetSw, fwdBuilder.add(context));
         rulePopulationCounter.incrementAndGet();
 
         return true;
@@ -419,12 +463,13 @@
      * @param routerIp the router ip
      * @return a collection of fwdobjective
      */
-    private Collection<ForwardingObjective> handleMpls(DeviceId targetSwId,
-                                                       DeviceId destSwId,
-                                                       Set<DeviceId> nextHops,
-                                                       int segmentId,
-                                                       IpAddress routerIp,
-                                                       boolean isMplsBos) {
+    private Collection<ForwardingObjective> handleMpls(
+                                        DeviceId targetSwId,
+                                        DeviceId destSwId,
+                                        Set<DeviceId> nextHops,
+                                        int segmentId,
+                                        IpAddress routerIp,
+                                        boolean isMplsBos) {
 
         TrafficSelector.Builder sbuilder = DefaultTrafficSelector.builder();
         List<ForwardingObjective.Builder> fwdObjBuilders = Lists.newArrayList();
@@ -443,7 +488,8 @@
         if (nextHops.size() == 1 && destSwId.equals(nextHops.toArray()[0])) {
             // If the next hop is the destination router for the segment, do pop
             log.debug("populateMplsRule: Installing MPLS forwarding objective for "
-                    + "label {} in switch {} with pop", segmentId, targetSwId);
+                    + "label {} in switch {} with pop to next-hops {}",
+                    segmentId, targetSwId, nextHops);
             // Not-bos pop case (php for the current label). If MPLS-ECMP
             // has been configured, the application we will request the
             // installation for an MPLS-ECMP group.
@@ -463,8 +509,9 @@
 
         } else {
             // next hop is not destination, SR CONTINUE case (swap with self)
-            log.debug("Installing MPLS forwarding objective for label {} in "
-                    + "switch {} without pop", segmentId, targetSwId);
+            log.debug("Installing MPLS forwarding objective for "
+                    + "label {} in switch {} without pop to next-hops {}",
+                    segmentId, targetSwId, nextHops);
             // Not-bos pop case. If MPLS-ECMP has been configured, the
             // application we will request the installation for an MPLS-ECMP
             // group.
@@ -522,7 +569,8 @@
      * @return true if all rules are set successfully, false otherwise
      */
     public boolean populateMplsRule(DeviceId targetSwId, DeviceId destSwId,
-                                    Set<DeviceId> nextHops, IpAddress routerIp) {
+                                    Set<DeviceId> nextHops,
+                                    IpAddress routerIp) {
 
         int segmentId;
         try {
@@ -563,7 +611,7 @@
     }
 
     private ForwardingObjective.Builder getMplsForwardingObjective(
-                                             DeviceId deviceId,
+                                             DeviceId targetSw,
                                              Set<DeviceId> nextHops,
                                              boolean phpRequired,
                                              boolean isBos,
@@ -600,33 +648,36 @@
         fwdBuilder.withTreatment(tbuilder.build());
         // if MPLS-ECMP == True we will build a standard NeighborSet.
         // Otherwise a RandomNeighborSet.
-        NeighborSet ns = NeighborSet.neighborSet(false, nextHops, false, destSw);
+        DestinationSet ns = DestinationSet.destinationSet(false, false, destSw);
         if (!isBos && this.srManager.getMplsEcmp()) {
-            ns = NeighborSet.neighborSet(false, nextHops, true, destSw);
+            ns = DestinationSet.destinationSet(false, true, destSw);
         } else if (!isBos && !this.srManager.getMplsEcmp()) {
-            ns = NeighborSet.neighborSet(true, nextHops, true, destSw);
+            ns = DestinationSet.destinationSet(true, true, destSw);
         }
+
         log.debug("Trying to get a nextObjId for mpls rule on device:{} to ns:{}",
-                  deviceId, ns);
+                  targetSw, ns);
+        DefaultGroupHandler gh = srManager.getGroupHandler(targetSw);
+        if (gh == null) {
+            log.warn("getNextObjectiveId query - groupHandler for device {} "
+                    + "not found", targetSw);
+            return null;
+        }
         // If BoS == True, all forwarding is via L3 ECMP group.
         // If Bos == False, the forwarding can be via MPLS-ECMP group or through
         // MPLS-Interface group. This depends on the configuration of the option
         // MPLS-ECMP.
         // The metadata informs the driver that the next-Objective will be used
         // by MPLS flows and if Bos == False the driver will use MPLS groups.
-        DefaultGroupHandler grpHandler = srManager.getGroupHandler(deviceId);
-        if (grpHandler == null) {
-            log.warn("populateIPRuleForRouter: groupHandler for device {} "
-                    + "not found", deviceId);
-            return null;
-        }
-        int nextId = grpHandler.getNextObjectiveId(ns, meta, isBos);
+        Map<DeviceId, Set<DeviceId>> dstNextHops = new HashMap<>();
+        dstNextHops.put(destSw, nextHops);
+        int nextId = gh.getNextObjectiveId(ns, dstNextHops, meta, isBos);
         if (nextId <= 0) {
-            log.warn("No next objective in {} for ns: {}", deviceId, ns);
+            log.warn("No next objective in {} for ns: {}", targetSw, ns);
             return null;
         } else {
             log.debug("nextObjId found:{} for mpls rule on device:{} to ns:{}",
-                      nextId, deviceId, ns);
+                      nextId, targetSw, ns);
         }
 
         fwdBuilder.nextStep(nextId);
@@ -775,11 +826,15 @@
      * @param deviceId the switch dpid for the router
      */
     public void populateIpPunts(DeviceId deviceId) {
-        Ip4Address routerIpv4;
-        Ip6Address routerIpv6;
+        Ip4Address routerIpv4, pairRouterIpv4 = null;
+        Ip6Address routerIpv6, pairRouterIpv6 = null;
         try {
             routerIpv4 = config.getRouterIpv4(deviceId);
             routerIpv6 = config.getRouterIpv6(deviceId);
+            if (config.isPairedEdge(deviceId)) {
+                pairRouterIpv4 = config.getRouterIpv4(config.getPairDeviceId(deviceId));
+                pairRouterIpv6 = config.getRouterIpv6(config.getPairDeviceId(deviceId));
+            }
         } catch (DeviceConfigNotFoundException e) {
             log.warn(e.getMessage() + " Aborting populateIpPunts.");
             return;
@@ -795,6 +850,12 @@
         if (routerIpv6 != null) {
             allIps.add(routerIpv6);
         }
+        if (pairRouterIpv4 != null) {
+            allIps.add(pairRouterIpv4);
+        }
+        if (pairRouterIpv6 != null) {
+            allIps.add(pairRouterIpv6);
+        }
         for (IpAddress ipaddr : allIps) {
             TrafficSelector.Builder sbuilder = buildIpSelectorFromIpAddress(ipaddr);
             Optional<DeviceId> optDeviceId = Optional.of(deviceId);
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
index bdc8aa8..1f126c6 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
@@ -84,8 +84,9 @@
 import org.onosproject.segmentrouting.config.SegmentRoutingAppConfig;
 import org.onosproject.segmentrouting.config.XConnectConfig;
 import org.onosproject.segmentrouting.grouphandler.DefaultGroupHandler;
-import org.onosproject.segmentrouting.grouphandler.NeighborSet;
-import org.onosproject.segmentrouting.storekey.NeighborSetNextObjectiveStoreKey;
+import org.onosproject.segmentrouting.grouphandler.DestinationSet;
+import org.onosproject.segmentrouting.grouphandler.NextNeighbors;
+import org.onosproject.segmentrouting.storekey.DestinationSetNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.PortNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.SubnetAssignedVidStoreKey;
 import org.onosproject.segmentrouting.storekey.VlanNextObjectiveStoreKey;
@@ -109,6 +110,7 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 import static com.google.common.base.Preconditions.checkState;
@@ -208,10 +210,10 @@
     private Map<DeviceId, DefaultGroupHandler> groupHandlerMap =
             new ConcurrentHashMap<>();
     /**
-     * Per device next objective ID store with (device id + neighbor set) as key.
+     * Per device next objective ID store with (device id + destination set) as key.
      */
-    EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey, Integer>
-            nsNextObjStore = null;
+    EventuallyConsistentMap<DestinationSetNextObjectiveStoreKey, NextNeighbors>
+            dsNextObjStore = null;
     /**
      * Per device next objective ID store with (device id + subnet) as key.
      */
@@ -236,6 +238,8 @@
     private EventuallyConsistentMap<String, Tunnel> tunnelStore = null;
     private EventuallyConsistentMap<String, Policy> policyStore = null;
 
+    private AtomicBoolean programmingScheduled = new AtomicBoolean();
+
     private final ConfigFactory<DeviceId, SegmentRoutingDeviceConfig> deviceConfigFactory =
             new ConfigFactory<DeviceId, SegmentRoutingDeviceConfig>(
                     SubjectFactories.DEVICE_SUBJECT_FACTORY,
@@ -307,14 +311,14 @@
         appId = coreService.registerApplication(APP_NAME);
 
         log.debug("Creating EC map nsnextobjectivestore");
-        EventuallyConsistentMapBuilder<NeighborSetNextObjectiveStoreKey, Integer>
+        EventuallyConsistentMapBuilder<DestinationSetNextObjectiveStoreKey, NextNeighbors>
                 nsNextObjMapBuilder = storageService.eventuallyConsistentMapBuilder();
-        nsNextObjStore = nsNextObjMapBuilder
+        dsNextObjStore = nsNextObjMapBuilder
                 .withName("nsnextobjectivestore")
                 .withSerializer(createSerializer())
                 .withTimestampProvider((k, v) -> new WallClockTimestamp())
                 .build();
-        log.trace("Current size {}", nsNextObjStore.size());
+        log.trace("Current size {}", dsNextObjStore.size());
 
         log.debug("Creating EC map vlannextobjectivestore");
         EventuallyConsistentMapBuilder<VlanNextObjectiveStoreKey, Integer>
@@ -399,10 +403,11 @@
     private KryoNamespace.Builder createSerializer() {
         return new KryoNamespace.Builder()
                 .register(KryoNamespaces.API)
-                .register(NeighborSetNextObjectiveStoreKey.class,
+                .register(DestinationSetNextObjectiveStoreKey.class,
                         VlanNextObjectiveStoreKey.class,
                         SubnetAssignedVidStoreKey.class,
-                        NeighborSet.class,
+                        DestinationSet.class,
+                        NextNeighbors.class,
                         Tunnel.class,
                         DefaultTunnel.class,
                         Policy.class,
@@ -436,7 +441,7 @@
         deviceListener = null;
         groupHandlerMap.clear();
 
-        nsNextObjStore.destroy();
+        dsNextObjStore.destroy();
         vlanNextObjStore.destroy();
         portNextObjStore.destroy();
         tunnelStore.destroy();
@@ -514,9 +519,9 @@
     }
 
     @Override
-    public ImmutableMap<NeighborSetNextObjectiveStoreKey, Integer> getNeighborSet() {
-        if (nsNextObjStore != null) {
-            return ImmutableMap.copyOf(nsNextObjStore.entrySet());
+    public ImmutableMap<DestinationSetNextObjectiveStoreKey, NextNeighbors> getDestinationSet() {
+        if (dsNextObjStore != null) {
+            return ImmutableMap.copyOf(dsNextObjStore.entrySet());
         } else {
             return ImmutableMap.of();
         }
@@ -541,12 +546,13 @@
     }
 
     /**
-     * Per device next objective ID store with (device id + neighbor set) as key.
+     * Per device next objective ID store with (device id + destination set) as key.
      *
      * @return next objective ID store
      */
-    public EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey, Integer> nsNextObjStore() {
-        return nsNextObjStore;
+    public EventuallyConsistentMap<DestinationSetNextObjectiveStoreKey, NextNeighbors>
+                dsNextObjStore() {
+        return dsNextObjStore;
     }
 
     /**
@@ -568,7 +574,8 @@
     }
 
     /**
-     * Returns the MPLS-ECMP configuration.
+     * Returns the MPLS-ECMP configuration which indicates whether ECMP on
+     * labeled packets should be programmed or not.
      *
      * @return MPLS-ECMP value
      */
@@ -792,6 +799,66 @@
         return false;
     }
 
+    /**
+     * Returns true if the link being queried is a bidirectional link. A bidi
+     * link is defined as a link, whose reverse link - ie. the link in the reverse
+     * direction - has been seen-before and is up. It is not necessary for the link
+     * being queried to be a seen-link.
+     *
+     * @param link the infrastructure link being queried
+     * @return true if another unidirectional link exists in the reverse direction,
+     *              has been seen-before and is up
+     */
+    public boolean isBidirectional(Link link) {
+        Link reverseLink = linkService.getLink(link.dst(), link.src());
+        if (reverseLink == null) {
+            return false;
+        }
+        Boolean result = isSeenLinkUp(reverseLink);
+        if (result == null) {
+            return false;
+        }
+        return result.booleanValue();
+    }
+
+    /**
+     * Determines if the given link should be avoided in routing calculations
+     * by policy or design.
+     *
+     * @param link the infrastructure link being queried
+     * @return true if link should be avoided
+     */
+    public boolean avoidLink(Link link) {
+        // XXX currently only avoids all pair-links. In the future can be
+        // extended to avoid any generic link
+        DeviceId src = link.src().deviceId();
+        PortNumber srcPort = link.src().port();
+        if (deviceConfiguration == null || !deviceConfiguration.isConfigured(src)) {
+            log.warn("Device {} not configured..cannot avoid link {}", src, link);
+            return false;
+        }
+        DeviceId pairDev;
+        PortNumber pairLocalPort, pairRemotePort = null;
+        try {
+            pairDev = deviceConfiguration.getPairDeviceId(src);
+            pairLocalPort = deviceConfiguration.getPairLocalPort(src);
+            if (pairDev != null) {
+                pairRemotePort = deviceConfiguration.getPairLocalPort(pairDev);
+            }
+        } catch (DeviceConfigNotFoundException e) {
+            log.warn("Pair dev for dev {} not configured..cannot avoid link {}",
+                     src, link);
+            return false;
+        }
+
+        if (srcPort.equals(pairLocalPort) &&
+                link.dst().deviceId().equals(pairDev) &&
+                link.dst().port().equals(pairRemotePort)) {
+            return true;
+        }
+        return false;
+    }
+
     private class InternalPacketProcessor implements PacketProcessor {
         @Override
         public void process(PacketContext context) {
@@ -807,7 +874,8 @@
                 return;
             }
 
-            log.trace("Rcvd pktin: {}", ethernet);
+            log.trace("Rcvd pktin from {}: {}", context.inPacket().receivedFrom(),
+                      ethernet);
             if (ethernet.getEtherType() == TYPE_ARP) {
                 log.warn("Received unexpected ARP packet on {}",
                          context.inPacket().receivedFrom());
@@ -982,7 +1050,7 @@
     private void processLinkAdded(Link link) {
         log.info("** LINK ADDED {}", link.toString());
         if (!deviceConfiguration.isConfigured(link.src().deviceId())) {
-            log.warn("Source device of this link is not configured.");
+            log.warn("Source device of this link is not configured..not processing");
             return;
         }
         if (link.type() != Link.Type.DIRECT) {
@@ -1014,16 +1082,39 @@
             }
         }
 
-        log.trace("Starting optimized route population process");
+        /*// process link only if it is bidirectional
+        if (!isBidirectional(link)) {
+            log.debug("Link not bidirectional.. waiting for other direction "
+                    + "src {} --> dst {} ", link.dst(), link.src());
+            // note that if we are not processing for routing, it should at least
+            // be considered a seen-link
+            updateSeenLink(link, true);
+            return;
+        }
+         TO DO this ensure that rehash is still done correctly even if link is
+         not processed for rerouting - perhaps rehash in both directions when
+         it ultimately becomes bidi?
+        */
+
+        log.debug("Starting optimized route population process for link "
+                + "{} --> {}", link.src(), link.dst());
         boolean seenBefore = isSeenLink(link);
         defaultRoutingHandler.populateRoutingRulesForLinkStatusChange(null, link, null);
+
+        // It's possible that linkUp causes no route-path change as ECMP graph does
+        // not change if the link is a parallel link (same src-dst as another link.
+        // However we still need to update ECMP hash groups to include new buckets
+        // for the link that has come up.
         if (mastershipService.isLocalMaster(link.src().deviceId())) {
-            if (!seenBefore) {
+            if (!seenBefore && isParallelLink(link)) {
                 // if link seen first time, we need to ensure hash-groups have all ports
+                log.debug("Attempting retryHash for paralled first-time link {}", link);
                 groupHandler.retryHash(link, false, true);
             } else {
                 //seen before-link
                 if (isParallelLink(link)) {
+                    log.debug("Attempting retryHash for paralled seen-before "
+                            + "link {}", link);
                     groupHandler.retryHash(link, false, false);
                 }
             }
@@ -1062,8 +1153,8 @@
         routingRulePopulator.populateArpNdpPunts(device.id());
 
         if (deviceConfiguration == null || !deviceConfiguration.isConfigured(device.id())) {
-            log.warn("Device configuration uploading. Device {} will be "
-                    + "processed after config completes.", device.id());
+            log.warn("Device configuration unavailable. Device {} will be "
+                    + "processed after configuration.", device.id());
             return;
         }
         processDeviceAddedInternal(device.id());
@@ -1110,10 +1201,10 @@
     }
 
     private void processDeviceRemoved(Device device) {
-        nsNextObjStore.entrySet().stream()
+        dsNextObjStore.entrySet().stream()
                 .filter(entry -> entry.getKey().deviceId().equals(device.id()))
                 .forEach(entry -> {
-                    nsNextObjStore.remove(entry.getKey());
+                    dsNextObjStore.remove(entry.getKey());
                 });
         vlanNextObjStore.entrySet().stream()
                 .filter(entry -> entry.getKey().deviceId().equals(device.id()))
@@ -1225,6 +1316,7 @@
     }
 
     private class InternalConfigListener implements NetworkConfigListener {
+        private static final long PROGRAM_DELAY = 2;
         SegmentRoutingManager srManager;
 
         /**
@@ -1240,8 +1332,11 @@
          * Reads network config and initializes related data structure accordingly.
          */
         public void configureNetwork() {
-
-            deviceConfiguration = new DeviceConfiguration(srManager);
+            if (deviceConfiguration == null) {
+                deviceConfiguration = new DeviceConfiguration(srManager);
+            } else {
+                deviceConfiguration.updateConfig();
+            }
 
             arpHandler = new ArpHandler(srManager);
             icmpHandler = new IcmpHandler(srManager);
@@ -1254,12 +1349,12 @@
             policyHandler = new PolicyHandler(appId, deviceConfiguration,
                                               flowObjectiveService,
                                               tunnelHandler, policyStore);
-
-            for (Device device : deviceService.getDevices()) {
-                processDeviceAdded(device);
+            // add a small delay to absorb multiple network config added notifications
+            if (!programmingScheduled.get()) {
+                programmingScheduled.set(true);
+                executorService.schedule(new ConfigChange(), PROGRAM_DELAY,
+                                         TimeUnit.SECONDS);
             }
-
-            defaultRoutingHandler.startPopulationProcess();
             mcastHandler.init();
         }
 
@@ -1340,6 +1435,17 @@
                 }
             }
         }
+
+        private final class ConfigChange implements Runnable {
+            @Override
+            public void run() {
+                programmingScheduled.set(false);
+                for (Device device : deviceService.getDevices()) {
+                    processDeviceAdded(device);
+                }
+                defaultRoutingHandler.startPopulationProcess();
+            }
+        }
     }
 
     private class InternalHostListener implements HostListener {
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/SegmentRoutingService.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/SegmentRoutingService.java
index 7f12adf..1cfff0c 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/SegmentRoutingService.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/SegmentRoutingService.java
@@ -17,7 +17,8 @@
 
 import org.onlab.packet.IpPrefix;
 import org.onosproject.net.DeviceId;
-import org.onosproject.segmentrouting.storekey.NeighborSetNextObjectiveStoreKey;
+import org.onosproject.segmentrouting.grouphandler.NextNeighbors;
+import org.onosproject.segmentrouting.storekey.DestinationSetNextObjectiveStoreKey;
 
 import com.google.common.collect.ImmutableMap;
 
@@ -128,9 +129,9 @@
     ImmutableMap<DeviceId, EcmpShortestPathGraph> getCurrentEcmpSpg();
 
     /**
-     * Returns the neighborSet-NextObjective store contents.
+     * Returns the destinatiomSet-NextObjective store contents.
      *
-     * @return current contents of the neighborSetNextObjectiveStore
+     * @return current contents of the destinationSetNextObjectiveStore
      */
-    ImmutableMap<NeighborSetNextObjectiveStoreKey, Integer> getNeighborSet();
+    ImmutableMap<DestinationSetNextObjectiveStoreKey, NextNeighbors> getDestinationSet();
 }
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/TunnelHandler.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/TunnelHandler.java
index de7663a..e18b920 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/TunnelHandler.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/TunnelHandler.java
@@ -20,7 +20,7 @@
 import org.onosproject.net.link.LinkService;
 import org.onosproject.segmentrouting.config.DeviceConfiguration;
 import org.onosproject.segmentrouting.grouphandler.DefaultGroupHandler;
-import org.onosproject.segmentrouting.grouphandler.NeighborSet;
+import org.onosproject.segmentrouting.grouphandler.DestinationSet;
 import org.onosproject.store.service.EventuallyConsistentMap;
 import org.slf4j.Logger;
 
@@ -222,7 +222,7 @@
             deviceIds.add(config.getDeviceId(sid));
         }
         // For these NeighborSet isMpls is meaningless.
-        NeighborSet ns = new NeighborSet(deviceIds, false,
+        DestinationSet ns = new DestinationSet(false,
                                          tunnel.labelIds().get(2),
                                          DeviceId.NONE);
 
@@ -234,7 +234,7 @@
             tunnel.allowToRemoveGroup(true);
         }
 
-        return groupHandlerMap.get(deviceId).getNextObjectiveId(ns, null, true);
+        return groupHandlerMap.get(deviceId).getNextObjectiveId(ns, null, null, true);
     }
 
 }
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/cli/NeighborSetCommand.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/cli/NeighborSetCommand.java
deleted file mode 100644
index d084cc3..0000000
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/cli/NeighborSetCommand.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2016-present Open Networking Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.onosproject.segmentrouting.cli;
-
-
-import java.util.Map;
-
-import org.apache.karaf.shell.commands.Command;
-import org.onosproject.cli.AbstractShellCommand;
-import org.onosproject.segmentrouting.SegmentRoutingService;
-import org.onosproject.segmentrouting.storekey.NeighborSetNextObjectiveStoreKey;
-
-/**
- * Command to read the current state of the neighborSetNextObjectiveStore.
- *
- */
-@Command(scope = "onos", name = "sr-ns-objstore",
-        description = "Displays the current neighborSet seen by each switch "
-                + "in the network and the next-id it maps to")
-public class NeighborSetCommand extends AbstractShellCommand {
-
-    private static final String FORMAT_MAPPING = "  %s";
-
-    @Override
-    protected void execute() {
-        SegmentRoutingService srService =
-                AbstractShellCommand.get(SegmentRoutingService.class);
-        printNeighborSet(srService.getNeighborSet());
-    }
-
-    private void printNeighborSet(Map<NeighborSetNextObjectiveStoreKey, Integer> ns) {
-        StringBuilder nsbldr = new StringBuilder();
-        ns.entrySet().forEach(entry -> {
-            nsbldr.append("\n " + entry.getKey());
-            nsbldr.append(" --> NextId: " + entry.getValue());
-        });
-        print(FORMAT_MAPPING, nsbldr.toString());
-    }
-}
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/cli/NextHopCommand.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/cli/NextHopCommand.java
new file mode 100644
index 0000000..2b43de5
--- /dev/null
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/cli/NextHopCommand.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2016-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.segmentrouting.cli;
+
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Map;
+
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.segmentrouting.SegmentRoutingService;
+import org.onosproject.segmentrouting.grouphandler.NextNeighbors;
+import org.onosproject.segmentrouting.storekey.DestinationSetNextObjectiveStoreKey;
+
+/**
+ * Command to read the current state of the DestinationSetNextObjectiveStore.
+ *
+ */
+@Command(scope = "onos", name = "sr-next-hops",
+        description = "Displays the current next-hops seen by each switch "
+                + "towards a set of destinations and the next-id it maps to")
+public class NextHopCommand extends AbstractShellCommand {
+
+    private static final String FORMAT_MAPPING = "  %s";
+
+    @Override
+    protected void execute() {
+        SegmentRoutingService srService =
+                AbstractShellCommand.get(SegmentRoutingService.class);
+        printDestinationSet(srService.getDestinationSet());
+    }
+
+    private void printDestinationSet(Map<DestinationSetNextObjectiveStoreKey,
+                                      NextNeighbors> ds) {
+        ArrayList<DestinationSetNextObjectiveStoreKey> a = new ArrayList<>();
+        ds.keySet().forEach(key -> a.add(key));
+        a.sort(new Comp());
+
+        StringBuilder dsbldr = new StringBuilder();
+        for (int i = 0; i < a.size(); i++) {
+            dsbldr.append("\n " + a.get(i));
+            dsbldr.append(" --> via: " + ds.get(a.get(i)));
+        }
+        print(FORMAT_MAPPING, dsbldr.toString());
+    }
+
+    static class Comp implements Comparator<DestinationSetNextObjectiveStoreKey> {
+
+        @Override
+        public int compare(DestinationSetNextObjectiveStoreKey o1,
+                           DestinationSetNextObjectiveStoreKey o2) {
+            int res = o1.deviceId().toString().compareTo(o2.deviceId().toString());
+            if (res < 0) {
+                return -1;
+            } else if (res > 0) {
+                return +1;
+            }
+            return 0;
+        }
+
+    }
+}
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/config/DeviceConfiguration.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/config/DeviceConfiguration.java
index 71a199e..8989288 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/config/DeviceConfiguration.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/config/DeviceConfiguration.java
@@ -72,6 +72,8 @@
         SetMultimap<PortNumber, IpAddress> gatewayIps;
         SetMultimap<PortNumber, IpPrefix> subnets;
         Map<Integer, Set<Integer>> adjacencySids;
+        DeviceId pairDeviceId;
+        PortNumber pairLocalPort;
 
         public SegmentRouterInfo() {
             gatewayIps = HashMultimap.create();
@@ -87,7 +89,10 @@
      */
     public DeviceConfiguration(SegmentRoutingManager srManager) {
         this.srManager = srManager;
+        updateConfig();
+    }
 
+    public void updateConfig() {
         // Read config from device subject, excluding gatewayIps and subnets.
         Set<DeviceId> deviceSubjects =
                 srManager.cfgService.getSubjects(DeviceId.class, SegmentRoutingDeviceConfig.class);
@@ -103,6 +108,8 @@
             info.mac = config.routerMac();
             info.isEdge = config.isEdgeRouter();
             info.adjacencySids = config.adjacencySids();
+            info.pairDeviceId = config.pairDeviceId();
+            info.pairLocalPort = config.pairLocalPort();
             deviceConfigMap.put(info.deviceId, info);
             log.debug("Read device config for device: {}", info.deviceId);
             /*
@@ -158,6 +165,7 @@
         });
     }
 
+
     @Override
     public boolean isConfigured(DeviceId deviceId) {
         return deviceConfigMap.get(deviceId) != null;
@@ -639,4 +647,34 @@
         }
         return false;
     }
+
+    public boolean isPairedEdge(DeviceId deviceId) throws DeviceConfigNotFoundException {
+        if (!isEdgeDevice(deviceId)) {
+            return false;
+        }
+        SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
+        return (srinfo.pairDeviceId == null) ? false : true;
+    }
+
+    public DeviceId getPairDeviceId(DeviceId deviceId) throws DeviceConfigNotFoundException {
+        SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
+        if (srinfo != null) {
+            return srinfo.pairDeviceId;
+        } else {
+            String message = "getPairDeviceId fails for device: " + deviceId + ".";
+            throw new DeviceConfigNotFoundException(message);
+        }
+    }
+
+    public PortNumber getPairLocalPort(DeviceId deviceId)
+            throws DeviceConfigNotFoundException {
+        SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
+        if (srinfo != null) {
+            return srinfo.pairLocalPort;
+        } else {
+            String message = "getPairLocalPort fails for device: " + deviceId + ".";
+            throw new DeviceConfigNotFoundException(message);
+        }
+    }
+
 }
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
index bfb1a8b..85b0c76 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
@@ -16,6 +16,7 @@
 package org.onosproject.segmentrouting.grouphandler;
 
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 
@@ -42,14 +43,13 @@
 import org.onosproject.segmentrouting.SegmentRoutingManager;
 import org.onosproject.segmentrouting.config.DeviceConfigNotFoundException;
 import org.onosproject.segmentrouting.config.DeviceProperties;
-import org.onosproject.segmentrouting.storekey.NeighborSetNextObjectiveStoreKey;
+import org.onosproject.segmentrouting.storekey.DestinationSetNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.PortNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.VlanNextObjectiveStoreKey;
 import org.onosproject.store.service.EventuallyConsistentMap;
 import org.slf4j.Logger;
 
 import java.net.URI;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -98,9 +98,9 @@
     protected ConcurrentHashMap<PortNumber, DeviceId> portDeviceMap =
             new ConcurrentHashMap<>();
 
-    // distributed store for (device+neighborset) mapped to next-id
-    protected EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey, Integer>
-            nsNextObjStore = null;
+    // distributed store for (device+destination-set) mapped to next-id and neighbors
+    protected EventuallyConsistentMap<DestinationSetNextObjectiveStoreKey, NextNeighbors>
+            dsNextObjStore = null;
     // distributed store for (device+subnet-ip-prefix) mapped to next-id
     protected EventuallyConsistentMap<VlanNextObjectiveStoreKey, Integer>
             vlanNextObjStore = null;
@@ -115,8 +115,8 @@
 
     protected KryoNamespace.Builder kryo = new KryoNamespace.Builder()
             .register(URI.class).register(HashSet.class)
-            .register(PortNumber.class)
-            .register(NeighborSet.class).register(PolicyGroupIdentifier.class)
+            .register(DeviceId.class).register(PortNumber.class)
+            .register(DestinationSet.class).register(PolicyGroupIdentifier.class)
             .register(PolicyGroupParams.class)
             .register(GroupBucketIdentifier.class)
             .register(GroupBucketIdentifier.BucketOutputType.class);
@@ -141,7 +141,7 @@
                     + " Skipping value assignment in DefaultGroupHandler");
         }
         this.flowObjectiveService = flowObjService;
-        this.nsNextObjStore = srManager.nsNextObjStore();
+        this.dsNextObjStore = srManager.dsNextObjStore();
         this.vlanNextObjStore = srManager.vlanNextObjStore();
         this.portNextObjStore = srManager.portNextObjStore();
         this.srManager = srManager;
@@ -231,28 +231,33 @@
             log.warn(e.getMessage() + " Aborting retryHash.");
             return;
         }
-        // find all the neighborSets related to link
-        Set<NeighborSet> nsSet = nsNextObjStore.keySet()
+        // find all the destinationSets related to link
+        Set<DestinationSetNextObjectiveStoreKey> dsKeySet = dsNextObjStore.entrySet()
                 .stream()
-                .filter((nsStoreEntry) -> (nsStoreEntry.deviceId().equals(deviceId)))
-                .map((nsStoreEntry) -> (nsStoreEntry.neighborSet()))
-                .filter((ns) -> (ns.getDeviceIds()
-                        .contains(link.dst().deviceId())))
+                .filter(entry -> entry.getKey().deviceId().equals(deviceId))
+                .filter(entry -> entry.getValue().containsNextHop(link.dst().deviceId()))
+                .map(entry -> entry.getKey())
                 .collect(Collectors.toSet());
-        log.debug("retryHash: nsNextObjStore contents for linkSrc {} -> linkDst {}: {}",
-                  deviceId, link.dst().deviceId(), nsSet);
 
-        for (NeighborSet ns : nsSet) {
-            Integer nextId = nsNextObjStore.
-                    get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-            if (nextId == null) {
+        log.debug("retryHash: dsNextObjStore contents for linkSrc {} -> linkDst {}: {}",
+                  deviceId, link.dst().deviceId(), dsKeySet);
+
+        for (DestinationSetNextObjectiveStoreKey dsKey : dsKeySet) {
+            NextNeighbors nextHops = dsNextObjStore.get(dsKey);
+            if (nextHops == null) {
                 log.warn("retryHash in device {}, but global store has no record "
-                        + "for neighbor-set {}", deviceId, ns);
+                         + "for dsKey:{}", deviceId, dsKey);
                 continue;
             }
+            int nextId = nextHops.nextId();
+            Set<DeviceId> dstSet = nextHops.getDstForNextHop(link.dst().deviceId());
             if (!linkDown) {
-                addToHashedNextObjective(link.src().port(), dstMac, ns,
-                                         nextId, false);
+                dstSet.forEach(dst -> {
+                    int edgeLabel = dsKey.destinationSet().getEdgeLabel(dst);
+                    addToHashedNextObjective(link.src().port(), dstMac,
+                                             edgeLabel, nextId, false);
+                });
+
                 if (firstTime) {
                     // some links may have come up before the next-objective was created
                     // we take this opportunity to ensure other ports to same next-hop-dst
@@ -262,12 +267,19 @@
                         if (p.equals(link.src().port())) {
                             continue;
                         }
-                        addToHashedNextObjective(p, dstMac, ns, nextId, false);
+                        dstSet.forEach(dst -> {
+                            int edgeLabel = dsKey.destinationSet().getEdgeLabel(dst);
+                            addToHashedNextObjective(p, dstMac, edgeLabel,
+                                                     nextId, false);
+                        });
                     }
                 }
             } else {
-                removeFromHashedNextObjective(link.src().port(), dstMac, ns,
-                                              nextId);
+                dstSet.forEach(dst -> {
+                    int edgeLabel = dsKey.destinationSet().getEdgeLabel(dst);
+                    removeFromHashedNextObjective(link.src().port(), dstMac,
+                                                  edgeLabel, nextId);
+                });
             }
         }
 
@@ -289,23 +301,23 @@
      *
      * @param outport port to add to hash group
      * @param dstMac destination mac address of next-hop
-     * @param ns neighbor set representing next-hops and destination switch
+     * @param edgeLabel the label to use in the bucket
      * @param nextId id for next-objective to which the bucket will be added
      * @param retry indicates if this method is being called on a retry attempt
      *              at adding a bucket to the group
      */
     private void addToHashedNextObjective(PortNumber outport, MacAddress dstMac,
-            NeighborSet ns, Integer nextId, boolean retry) {
+            int edgeLabel, Integer nextId, boolean retry) {
         // Create the new bucket to be updated
         TrafficTreatment.Builder tBuilder =
                 DefaultTrafficTreatment.builder();
         tBuilder.setOutput(outport)
             .setEthDst(dstMac)
             .setEthSrc(nodeMacAddr);
-        if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
+        if (edgeLabel != DestinationSet.NO_EDGE_LABEL) {
             tBuilder.pushMpls()
                 .copyTtlOut()
-                .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
+                .setMpls(MplsLabel.mplsLabel(edgeLabel));
         }
         // setup metadata to pass to nextObjective - indicate the vlan on egress
         // if needed by the switch pipeline. Since hashed next-hops are always to
@@ -319,9 +331,9 @@
                 .addTreatment(tBuilder.build())
                 .withMeta(metabuilder.build())
                 .fromApp(appId);
-        log.debug("{} in device {}: Adding Bucket with Port {} to next object id {}",
+        log.debug("{} in device {}: Adding Bucket with port/label {}/{} to nextId {}",
                  (retry) ? "retry-addToHash" : "addToHash",
-                         deviceId, outport, nextId);
+                         deviceId, outport, edgeLabel, nextId);
 
         ObjectiveContext context = new DefaultObjectiveContext(
                 (objective) -> log.debug("{} addedTo NextObj {} on {}",
@@ -341,21 +353,21 @@
     *
     * @param port port to remove from hash group
     * @param dstMac destination mac address of next-hop
-    * @param ns neighbor set representing next-hops and destination switch
+    * @param edgeLabel the label to use in the bucket
     * @param nextId id for next-objective from which the bucket will be removed
     */
    private void removeFromHashedNextObjective(PortNumber port, MacAddress dstMac,
-                                              NeighborSet ns, Integer nextId) {
+                                              int edgeLabel, Integer nextId) {
        // Create the bucket to be removed
        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
                .builder();
        tBuilder.setOutput(port)
            .setEthDst(dstMac)
            .setEthSrc(nodeMacAddr);
-       if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
+       if (edgeLabel != DestinationSet.NO_EDGE_LABEL) {
            tBuilder.pushMpls()
                .copyTtlOut()
-               .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
+               .setMpls(MplsLabel.mplsLabel(edgeLabel));
        }
        log.info("{} in device {}: Removing Bucket with Port {} to next object id {}",
                 "removeFromHash", deviceId, port, nextId);
@@ -395,20 +407,22 @@
     public boolean fixHashGroups(DeviceId targetSw, Set<DeviceId> nextHops,
                                  DeviceId destSw, boolean revoke) {
         // temporary storage of keys to be updated
-        Map<NeighborSetNextObjectiveStoreKey, Set<DeviceId>> tempStore =
+        Map<DestinationSetNextObjectiveStoreKey, Set<DeviceId>> tempStore =
                 new HashMap<>();
         boolean foundNextObjective = false;
 
-        // retrieve hash-groups meant for destSw, which have neighborSets
+        // retrieve hash-groups meant for destSw, which have destinationSets
         // with different neighbors than the given next-hops
-        for (NeighborSetNextObjectiveStoreKey nskey : nsNextObjStore.keySet()) {
-            if (!nskey.deviceId().equals(targetSw) ||
-                    !nskey.neighborSet().getDestinationSw().equals(destSw)) {
+        for (DestinationSetNextObjectiveStoreKey dskey : dsNextObjStore.keySet()) {
+            if (!dskey.deviceId().equals(targetSw) ||
+                    !dskey.destinationSet().getDestinationSwitches().contains(destSw)) {
                 continue;
             }
             foundNextObjective = true;
-            Set<DeviceId> currNeighbors = nskey.neighborSet().getDeviceIds();
-            Integer nextId = nsNextObjStore.get(nskey);
+            NextNeighbors nhops = dsNextObjStore.get(dskey);
+            Set<DeviceId> currNeighbors = nhops.nextHops(destSw);
+            int edgeLabel = dskey.destinationSet().getEdgeLabel(destSw);
+            Integer nextId = nhops.nextId();
 
             Set<DeviceId> diff;
             if (revoke) {
@@ -428,7 +442,7 @@
                     dstMac = deviceConfig.getDeviceMac(neighbor);
                 } catch (DeviceConfigNotFoundException e) {
                     log.warn(e.getMessage() + " Aborting fixHashGroup for nextId:"
-                            + nskey);
+                            + nextId);
                     return false;
                 }
                 if (devicePortMap.get(neighbor) == null ||
@@ -444,22 +458,22 @@
                                 + "with Port {} to next object id {}",
                                 deviceId, port, nextId);
                         removeFromHashedNextObjective(port, dstMac,
-                                                      nskey.neighborSet(),
+                                                      edgeLabel,
                                                       nextId);
                     }
                     // to update neighbor set with changes made
-                    tempStore.put(nskey, Sets.difference(currNeighbors, diff));
+                    tempStore.put(dskey, Sets.difference(currNeighbors, diff));
                 } else {
                     for (PortNumber port : devicePortMap.get(neighbor)) {
                         log.info("fixHashGroup in device {}: Adding Bucket "
                                 + "with Port {} to next object id {}",
                                 deviceId, port, nextId);
                         addToHashedNextObjective(port, dstMac,
-                                                 nskey.neighborSet(),
+                                                 edgeLabel,
                                                  nextId, false);
                     }
                     // to update neighbor set with changes made
-                    tempStore.put(nskey, Sets.union(currNeighbors, diff));
+                    tempStore.put(dskey, Sets.union(currNeighbors, diff));
                 }
             }
         }
@@ -470,30 +484,102 @@
             return true; // nothing to do, return true so ECMPspg is updated
         }
 
-        // update the nsNextObjectiveStore with new neighborSets to nextId mappings
-        for (NeighborSetNextObjectiveStoreKey oldkey : tempStore.keySet()) {
-            Integer nextId = nsNextObjStore.get(oldkey);
-            if (nextId == null) {
+        // update the dsNextObjectiveStore with new destinationSet to nextId mappings
+        for (DestinationSetNextObjectiveStoreKey key : tempStore.keySet()) {
+            NextNeighbors oldHops = dsNextObjStore.get(key);
+            if (oldHops == null) {
                 continue;
             }
-            Set<DeviceId> newNeighbors = tempStore.get(oldkey);
-            NeighborSet newNs = new NeighborSet(newNeighbors,
-                                                oldkey.neighborSet().mplsSet(),
-                                                oldkey.neighborSet().getEdgeLabel(),
-                                                oldkey.neighborSet().getDestinationSw());
-            NeighborSetNextObjectiveStoreKey newkey =
-                    new NeighborSetNextObjectiveStoreKey(deviceId, newNs);
-            log.debug("Updating nsNextObjStore: oldKey:{} -> newKey:{} :: nextId:{}",
-                      oldkey, newkey, nextId);
-            synchronized (nsNextObjStore) {
-                nsNextObjStore.remove(oldkey);
-                nsNextObjStore.put(newkey, nextId);
-            }
+            Set<DeviceId> newNeighbors = tempStore.get(key);
+            Set<DeviceId> oldNeighbors = ImmutableSet.copyOf(oldHops.nextHops(destSw));
+            oldHops.dstNextHops().put(destSw, newNeighbors);
+            log.debug("Updating nsNextObjStore: oldHops:{} -> newHops:{} :: nextId:{}",
+                      oldNeighbors, newNeighbors, oldHops.nextId());
         }
 
         return true;
     }
 
+
+    public boolean updateNextHops(DestinationSet ds,
+                                  Map<DeviceId, Set<DeviceId>> newDstNextHops) {
+        DestinationSetNextObjectiveStoreKey key =
+                new DestinationSetNextObjectiveStoreKey(deviceId, ds);
+        NextNeighbors currNext = dsNextObjStore.get(key);
+        Map<DeviceId, Set<DeviceId>> currDstNextHops = currNext.dstNextHops();
+
+        // add newDstNextHops to currDstNextHops for each dst
+        boolean success = true;
+        for (DeviceId dstSw : ds.getDestinationSwitches()) {
+            Set<DeviceId> currNhops = currDstNextHops.get(dstSw);
+            Set<DeviceId> newNhops = newDstNextHops.get(dstSw);
+            currNhops = (currNhops == null) ? Sets.newHashSet() : currNhops;
+            newNhops = (newNhops == null) ? Sets.newHashSet() : newNhops;
+            int edgeLabel = ds.getEdgeLabel(dstSw);
+            int nextId = currNext.nextId();
+
+            // new next hops should be added
+            boolean suc = updateAllPortsToNextHop(Sets.difference(newNhops, currNhops),
+                                                  edgeLabel, nextId, false);
+            if (suc) {
+                currNhops.addAll(newNhops);
+                currDstNextHops.put(dstSw, currNhops); // this is only a local change
+            }
+            success &= suc;
+        }
+
+        if (success) {
+            // update global store
+            dsNextObjStore.put(key, new NextNeighbors(currDstNextHops,
+                                                      currNext.nextId()));
+            log.debug("Updated device:{} ds:{} new next-hops: {}", deviceId, ds,
+                      dsNextObjStore.get(key));
+        }
+        return success;
+    }
+
+    private boolean updateAllPortsToNextHop(Set<DeviceId> diff, int edgeLabel,
+                                         int nextId, boolean revoke) {
+        for (DeviceId neighbor : diff) {
+            MacAddress dstMac;
+            try {
+                dstMac = deviceConfig.getDeviceMac(neighbor);
+            } catch (DeviceConfigNotFoundException e) {
+                log.warn(e.getMessage() + " Aborting fixHashGroup for nextId:"
+                        + nextId);
+                return false;
+            }
+            if (devicePortMap.get(neighbor) == null ||
+                    devicePortMap.get(neighbor).isEmpty()) {
+                log.warn("No ports found in dev:{} for neighbor:{} .. cannot "
+                        + "fix hash group for nextId: {}",
+                         deviceId, neighbor, nextId);
+                return false;
+            }
+            if (revoke) {
+                for (PortNumber port : devicePortMap.get(neighbor)) {
+                    log.debug("fixHashGroup in device {}: Removing Bucket "
+                            + "with Port {} edgeLabel:{} to next object id {}",
+                            deviceId, port, edgeLabel, nextId);
+                    removeFromHashedNextObjective(port, dstMac,
+                                                  edgeLabel,
+                                                  nextId);
+                }
+            } else {
+                for (PortNumber port : devicePortMap.get(neighbor)) {
+                    log.debug("fixHashGroup in device {}: Adding Bucket "
+                            + "with Port {} edgeLabel: {} to next object id {}",
+                            deviceId, port, edgeLabel, nextId);
+                    addToHashedNextObjective(port, dstMac,
+                                             edgeLabel,
+                                             nextId, false);
+                }
+            }
+        }
+        return true;
+    }
+
+
     /**
      * Adds or removes a port that has been configured with a vlan to a broadcast group
      * for bridging. Should only be called by the master instance for this device.
@@ -558,45 +644,61 @@
     }
 
     /**
-     * Returns the next objective of type hashed associated with the neighborset.
-     * If there is no next objective for this neighborset, this method
-     * would create a next objective and return. Optionally metadata can be
+     * Returns the next objective of type hashed associated with the destination set.
+     * In addition, updates the existing next-objective if new route-route paths found
+     * have resulted in the addition of new next-hops to a particular destination.
+     * If there is no existing next objective for this destination set, this method
+     * would create a next objective and return the nextId. Optionally metadata can be
      * passed in for the creation of the next objective.
      *
-     * @param ns neighborset
+     * @param ds destination set
+     * @param nextHops a map of per destination next hops
      * @param meta metadata passed into the creation of a Next Objective
      * @param isBos if Bos is set
      * @return int if found or -1 if there are errors in the creation of the
      *          neighbor set.
      */
-    public int getNextObjectiveId(NeighborSet ns, TrafficSelector meta, boolean isBos) {
-        Integer nextId = nsNextObjStore.
-                get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-        if (nextId == null) {
-            log.trace("getNextObjectiveId in device{}: Next objective id "
-                    + "not found for {} and creating", deviceId, ns);
+    public int getNextObjectiveId(DestinationSet ds,
+                                  Map<DeviceId, Set<DeviceId>> nextHops,
+                                  TrafficSelector meta, boolean isBos) {
+        NextNeighbors next = dsNextObjStore.
+                get(new DestinationSetNextObjectiveStoreKey(deviceId, ds));
+        if (next == null) {
+            log.debug("getNextObjectiveId in device{}: Next objective id "
+                    + "not found for {} ... creating", deviceId, ds);
             log.trace("getNextObjectiveId: nsNextObjStore contents for device {}: {}",
                       deviceId,
-                      nsNextObjStore.entrySet()
+                      dsNextObjStore.entrySet()
                       .stream()
                       .filter((nsStoreEntry) ->
                       (nsStoreEntry.getKey().deviceId().equals(deviceId)))
                       .collect(Collectors.toList()));
-            createGroupsFromNeighborsets(Collections.singleton(ns), meta, isBos);
-            nextId = nsNextObjStore.
-                    get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-            if (nextId == null) {
+
+            createGroupFromDestinationSet(ds, nextHops, meta, isBos);
+            next = dsNextObjStore.
+                    get(new DestinationSetNextObjectiveStoreKey(deviceId, ds));
+            if (next == null) {
                 log.warn("getNextObjectiveId: unable to create next objective");
+                // failure in creating group
                 return -1;
             } else {
                 log.debug("getNextObjectiveId in device{}: Next objective id {} "
-                    + "created for {}", deviceId, nextId, ns);
+                    + "created for {}", deviceId, next.nextId(), ds);
             }
         } else {
             log.trace("getNextObjectiveId in device{}: Next objective id {} "
-                    + "found for {}", deviceId, nextId, ns);
+                    + "found for {}", deviceId, next.nextId(), ds);
+            // should fix hash groups too if next-hops have changed
+            if (!next.dstNextHops().equals(nextHops)) {
+                log.debug("Nexthops have changed for dev:{} nextId:{} ..updating",
+                          deviceId, next.nextId());
+                if (!updateNextHops(ds, nextHops)) {
+                    // failure in updating group
+                    return -1;
+                }
+            }
         }
-        return nextId;
+        return next.nextId();
     }
 
     /**
@@ -663,10 +765,10 @@
      * @param ns neighbor set to check
      * @return true if it exists, false otherwise
      */
-    public boolean hasNextObjectiveId(NeighborSet ns) {
-        Integer nextId = nsNextObjStore.
-                get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-        if (nextId == null) {
+    public boolean hasNextObjectiveId(DestinationSet ns) {
+        NextNeighbors nextHops = dsNextObjStore.
+                get(new DestinationSetNextObjectiveStoreKey(deviceId, ns));
+        if (nextHops == null) {
             return false;
         }
 
@@ -704,100 +806,59 @@
         }
     }
 
-    protected Set<Set<DeviceId>> getPowerSetOfNeighbors(Set<DeviceId> neighbors) {
-        List<DeviceId> list = new ArrayList<>(neighbors);
-        Set<Set<DeviceId>> sets = new HashSet<>();
-        // get the number of elements in the neighbors
-        int elements = list.size();
-        // the number of members of a power set is 2^n
-        // including the empty set
-        int powerElements = (1 << elements);
-
-        // run a binary counter for the number of power elements
-        // NOTE: Exclude empty set
-        for (long i = 1; i < powerElements; i++) {
-            Set<DeviceId> neighborSubSet = new HashSet<>();
-            for (int j = 0; j < elements; j++) {
-                if ((i >> j) % 2 == 1) {
-                    neighborSubSet.add(list.get(j));
-                }
-            }
-            sets.add(neighborSubSet);
-        }
-        return sets;
-    }
-
-    private boolean isSegmentIdSameAsNodeSegmentId(DeviceId deviceId, int sId) {
-        int segmentId;
-        try {
-            // IPv6 sid is not inserted. this part of the code is not used for now.
-            segmentId = deviceConfig.getIPv4SegmentId(deviceId);
-        } catch (DeviceConfigNotFoundException e) {
-            log.warn(e.getMessage() + " Aborting isSegmentIdSameAsNodeSegmentId.");
-            return false;
-        }
-
-        return segmentId == sId;
-    }
-
-    protected List<Integer> getSegmentIdsTobePairedWithNeighborSet(Set<DeviceId> neighbors) {
-
-        List<Integer> nsSegmentIds = new ArrayList<>();
-
-        // Always pair up with no edge label
-        // If (neighbors.size() == 1) {
-        nsSegmentIds.add(-1);
-        // }
-
-        // Filter out SegmentIds matching with the
-        // nodes in the combo
-        for (Integer sId : allSegmentIds) {
-            if (sId.equals(this.ipv4NodeSegmentId)) {
-                continue;
-            }
-            boolean filterOut = false;
-            // Check if the edge label being set is of
-            // any node in the Neighbor set
-            for (DeviceId deviceId : neighbors) {
-                if (isSegmentIdSameAsNodeSegmentId(deviceId, sId)) {
-                    filterOut = true;
-                    break;
-                }
-            }
-            if (!filterOut) {
-                nsSegmentIds.add(sId);
-            }
-        }
-        return nsSegmentIds;
-    }
-
     /**
-     * Creates hash groups from a set of NeighborSet given.
+     * Creates a NextObjective for a hash group in this device from a given
+     * DestinationSet.
      *
-     * @param nsSet a set of NeighborSet
+     * @param ds the DestinationSet
+     * @param neighbors a map for each destination and its next-hops
      * @param meta metadata passed into the creation of a Next Objective
      * @param isBos if BoS is set
      */
-    public void createGroupsFromNeighborsets(Set<NeighborSet> nsSet,
-                                             TrafficSelector meta,
-                                             boolean isBos) {
-        for (NeighborSet ns : nsSet) {
-            int nextId = flowObjectiveService.allocateNextId();
-            NextObjective.Type type = NextObjective.Type.HASHED;
-            Set<DeviceId> neighbors = ns.getDeviceIds();
-            // If Bos == False and MPLS-ECMP == false, we have
-            // to use simple group and we will pick a single neighbor.
-            if (!isBos && !srManager.getMplsEcmp()) {
-                type = NextObjective.Type.SIMPLE;
-                neighbors = Collections.singleton(ns.getFirstNeighbor());
+    public void createGroupFromDestinationSet(DestinationSet ds,
+                                              Map<DeviceId, Set<DeviceId>> neighbors,
+                                              TrafficSelector meta,
+                                              boolean isBos) {
+        int nextId = flowObjectiveService.allocateNextId();
+        NextObjective.Type type = NextObjective.Type.HASHED;
+        if (neighbors == null || neighbors.isEmpty()) {
+            log.warn("createGroupsFromDestinationSet: needs at least one neighbor"
+                    + "to create group in dev:{} for ds: {} with next-hops {}",
+                    deviceId, ds, neighbors);
+            return;
+        }
+        // If Bos == False and MPLS-ECMP == false, we have
+        // to use simple group and we will pick a single neighbor for a single dest.
+        if (!isBos && !srManager.getMplsEcmp()) {
+            type = NextObjective.Type.SIMPLE;
+        }
+
+        NextObjective.Builder nextObjBuilder = DefaultNextObjective
+                .builder()
+                .withId(nextId)
+                .withType(type)
+                .fromApp(appId);
+        if (meta != null) {
+            nextObjBuilder.withMeta(meta);
+        }
+
+        // create treatment buckets for each neighbor for each dst Device
+        // except in the special case where we only want to pick a single
+        // neighbor for a simple group
+        boolean foundSingleNeighbor = false;
+        boolean treatmentAdded = false;
+        Map<DeviceId, Set<DeviceId>> dstNextHops = new ConcurrentHashMap<>();
+        for (DeviceId dst : ds.getDestinationSwitches()) {
+            Set<DeviceId> nextHops = neighbors.get(dst);
+            if (nextHops == null || nextHops.isEmpty()) {
+                continue;
             }
-            NextObjective.Builder nextObjBuilder = DefaultNextObjective
-                    .builder()
-                    .withId(nextId)
-                    .withType(type)
-                    .fromApp(appId);
-            // For each neighbor, we have to update the sent actions
-            for (DeviceId neighborId : neighbors) {
+
+            if (foundSingleNeighbor) {
+                break;
+            }
+
+            for (DeviceId neighborId : nextHops) {
                 if (devicePortMap.get(neighborId) == null) {
                     log.warn("Neighbor {} is not in the port map yet for dev:{}",
                              neighborId, deviceId);
@@ -812,53 +873,70 @@
                 try {
                     neighborMac = deviceConfig.getDeviceMac(neighborId);
                 } catch (DeviceConfigNotFoundException e) {
-                    log.warn(e.getMessage() + " Aborting createGroupsFromNeighborsets.");
+                    log.warn(e.getMessage() + " Aborting createGroupsFromDestinationset.");
                     return;
                 }
-                // For each port, we have to create a new treatment
+                // For each port to the neighbor, we create a new treatment
                 Set<PortNumber> neighborPorts = devicePortMap.get(neighborId);
                 // In this case we are using a SIMPLE group. We randomly pick a port
                 if (!isBos && !srManager.getMplsEcmp()) {
                     int size = devicePortMap.get(neighborId).size();
                     int index = RandomUtils.nextInt(0, size);
                     neighborPorts = Collections.singleton(
-                            Iterables.get(devicePortMap.get(neighborId), index)
-                    );
+                                        Iterables.get(devicePortMap.get(neighborId),
+                                                      index));
+                    foundSingleNeighbor = true;
                 }
                 for (PortNumber sp : neighborPorts) {
                     TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
                             .builder();
-                    tBuilder.setEthDst(neighborMac)
-                            .setEthSrc(nodeMacAddr);
-                    if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
+                    tBuilder.setEthDst(neighborMac).setEthSrc(nodeMacAddr);
+                    int edgeLabel = ds.getEdgeLabel(dst);
+                    if (edgeLabel != DestinationSet.NO_EDGE_LABEL) {
                         tBuilder.pushMpls()
-                                .copyTtlOut()
-                                .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
+                        .copyTtlOut()
+                        .setMpls(MplsLabel.mplsLabel(edgeLabel));
                     }
                     tBuilder.setOutput(sp);
                     nextObjBuilder.addTreatment(tBuilder.build());
+                    treatmentAdded = true;
+                    //update store
+                    Set<DeviceId> existingNeighbors = dstNextHops.get(dst);
+                    if (existingNeighbors == null) {
+                        existingNeighbors = new HashSet<>();
+                    }
+                    existingNeighbors.add(neighborId);
+                    dstNextHops.put(dst, existingNeighbors);
+                    log.debug("creating treatment for port/label {}/{} in next:{}",
+                              sp, edgeLabel, nextId);
+                }
+
+                if (foundSingleNeighbor) {
+                    break;
                 }
             }
-            if (meta != null) {
-                nextObjBuilder.withMeta(meta);
-            }
-
-            ObjectiveContext context = new DefaultObjectiveContext(
-                    (objective) ->
-                            log.debug("createGroupsFromNeighborsets installed "
-                                    + "NextObj {} on {}", nextId, deviceId),
-                    (objective, error) ->
-                            log.warn("createGroupsFromNeighborsets failed to install"
-                                    + " NextObj {} on {}: {}", nextId, deviceId, error)
-                    );
-            NextObjective nextObj = nextObjBuilder.add(context);
-            log.debug("**createGroupsFromNeighborsets: Submitted "
-                    + "next objective {} in device {}",
-                    nextId, deviceId);
-            flowObjectiveService.next(deviceId, nextObj);
-            nsNextObjStore.put(new NeighborSetNextObjectiveStoreKey(deviceId, ns),
-                               nextId);
         }
+
+        if (!treatmentAdded) {
+            log.warn("Could not createGroup from DestinationSet {} without any"
+                    + "next hops {}", ds, neighbors);
+            return;
+        }
+        ObjectiveContext context = new DefaultObjectiveContext(
+                (objective) ->
+                log.debug("createGroupsFromDestinationSet installed "
+                        + "NextObj {} on {}", nextId, deviceId),
+                (objective, error) ->
+                log.warn("createGroupsFromDestinationSet failed to install"
+                        + " NextObj {} on {}: {}", nextId, deviceId, error)
+                );
+        NextObjective nextObj = nextObjBuilder.add(context);
+        log.debug(".. createGroupsFromDestinationSet: Submitted "
+                + "next objective {} in device {}", nextId, deviceId);
+        flowObjectiveService.next(deviceId, nextObj);
+        //update store
+        dsNextObjStore.put(new DestinationSetNextObjectiveStoreKey(deviceId, ds),
+                           new NextNeighbors(dstNextHops, nextId));
     }
 
     /**
@@ -929,7 +1007,8 @@
      * @return true if the vlan id is not contained in any vlanTagged config
      */
     private boolean toPopVlan(PortNumber portNumber, VlanId vlanId) {
-        return srManager.interfaceService.getInterfacesByPort(new ConnectPoint(deviceId, portNumber))
+        return srManager.interfaceService
+                .getInterfacesByPort(new ConnectPoint(deviceId, portNumber))
                 .stream().noneMatch(intf -> intf.vlanTagged().contains(vlanId));
     }
 
@@ -977,8 +1056,11 @@
      * @return true if succeeds, false otherwise
      */
     public boolean removeGroup(int objectiveId) {
-
-        if (nsNextObjStore.containsValue(objectiveId)) {
+        for (Map.Entry<DestinationSetNextObjectiveStoreKey, NextNeighbors> e :
+                dsNextObjStore.entrySet()) {
+            if (e.getValue().nextId() != objectiveId) {
+                continue;
+            }
             NextObjective.Builder nextObjBuilder = DefaultNextObjective
                     .builder().withId(objectiveId)
                     .withType(NextObjective.Type.HASHED).fromApp(appId);
@@ -994,12 +1076,7 @@
                     objectiveId, deviceId);
             flowObjectiveService.next(deviceId, nextObjective);
 
-            for (Map.Entry<NeighborSetNextObjectiveStoreKey, Integer> entry: nsNextObjStore.entrySet()) {
-                if (entry.getValue().equals(objectiveId)) {
-                    nsNextObjStore.remove(entry.getKey());
-                    break;
-                }
-            }
+            dsNextObjStore.remove(e.getKey());
             return true;
         }
 
@@ -1009,10 +1086,10 @@
     /**
      * Removes all groups from all next objective stores.
      */
-    public void removeAllGroups() {
-        for (Map.Entry<NeighborSetNextObjectiveStoreKey, Integer> entry:
+    /*public void removeAllGroups() {
+        for (Map.Entry<NeighborSetNextObjectiveStoreKey, NextNeighbors> entry:
                 nsNextObjStore.entrySet()) {
-            removeGroup(entry.getValue());
+            removeGroup(entry.getValue().nextId());
         }
         for (Map.Entry<PortNextObjectiveStoreKey, Integer> entry:
                 portNextObjStore.entrySet()) {
@@ -1022,8 +1099,8 @@
                 vlanNextObjStore.entrySet()) {
             removeGroup(entry.getValue());
         }
-        // should probably clean local stores port-neighbor
-    }
+    }*/ //XXX revisit
+
 
     /**
      * RetryHashBkts is a one-time retry at populating all the buckets of a
@@ -1042,21 +1119,24 @@
         @Override
         public void run() {
             log.debug("RETRY Hash buckets for linkup: {}", link);
-            Set<NeighborSet> nsSet = nsNextObjStore.keySet()
+            Set<DestinationSetNextObjectiveStoreKey> dsKeySet = dsNextObjStore.entrySet()
                     .stream()
-                    .filter(nsStoreEntry -> nsStoreEntry.deviceId().equals(deviceId))
-                    .map(nsStoreEntry -> nsStoreEntry.neighborSet())
-                    .filter(ns -> ns.getDeviceIds()
-                            .contains(link.dst().deviceId()))
+                    .filter(entry -> entry.getKey().deviceId().equals(deviceId))
+                    .filter(entry -> entry.getValue().containsNextHop(link.dst().deviceId()))
+                    .map(entry -> entry.getKey())
                     .collect(Collectors.toSet());
-            log.debug("retry-link: nsNextObjStore contents for device {}: {}",
-                      deviceId, nsSet);
-            for (NeighborSet ns : nsSet) {
-                Integer nextId = nsNextObjStore.
-                        get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-                if (nextId != null) {
-                    addToHashedNextObjective(link.src().port(), dstMac, ns,
-                                             nextId, true);
+
+            log.debug("retry-link: dsNextObjStore contents for device {}: {}",
+                      deviceId, dsKeySet);
+            for (DestinationSetNextObjectiveStoreKey dsKey : dsKeySet) {
+                NextNeighbors next = dsNextObjStore.get(dsKey);
+                if (next != null) {
+                    Set<DeviceId> dstSet = next.getDstForNextHop(link.dst().deviceId());
+                    dstSet.forEach(dst -> {
+                        int edgeLabel = dsKey.destinationSet().getEdgeLabel(dst);
+                        addToHashedNextObjective(link.src().port(), dstMac, edgeLabel,
+                                                 next.nextId(), true);
+                    });
                 }
             }
         }
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DestinationSet.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DestinationSet.java
new file mode 100644
index 0000000..7b0864a
--- /dev/null
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/DestinationSet.java
@@ -0,0 +1,239 @@
+/*
+ * Copyright 2015-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.segmentrouting.grouphandler;
+
+import org.onosproject.net.DeviceId;
+import org.slf4j.Logger;
+
+import com.google.common.base.MoreObjects.ToStringHelper;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Representation of a set of destination switch dpids along with their edge-node
+ * labels. Meant to be used as a lookup-key in a hash-map to retrieve an ECMP-group
+ * that hashes packets towards a specific destination switch,
+ * or paired-destination switches.
+ */
+public class DestinationSet {
+    public static final int NO_EDGE_LABEL = -1;
+    private static final int NOT_ASSIGNED = 0;
+    private boolean mplsSet;
+    private final DeviceId dstSw1;
+    private final int edgeLabel1;
+    private final DeviceId dstSw2;
+    private final int edgeLabel2;
+
+
+    protected static final Logger log = getLogger(DestinationSet.class);
+
+    /**
+     * Constructor for a single destination with no Edge label.
+     *
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param dstSw the destination switch
+     */
+    public DestinationSet(boolean isMplsSet, DeviceId dstSw) {
+        this.edgeLabel1 = NO_EDGE_LABEL;
+        this.mplsSet = isMplsSet;
+        this.dstSw1 = dstSw;
+        this.edgeLabel2 = NOT_ASSIGNED;
+        this.dstSw2 = null;
+    }
+
+    /**
+     * Constructor for a single destination with Edge label.
+     *
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param edgeLabel label to be pushed as part of group operation
+     * @param dstSw the destination switch
+     */
+    public DestinationSet(boolean isMplsSet,
+                       int edgeLabel, DeviceId dstSw) {
+        this.mplsSet = isMplsSet;
+        this.edgeLabel1 = edgeLabel;
+        this.dstSw1 = dstSw;
+        this.edgeLabel2 = NOT_ASSIGNED;
+        this.dstSw2 = null;
+    }
+
+    /**
+     * Constructor for paired destination switches and their associated
+     * edge labels.
+     *
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param edgeLabel1 label to be pushed as part of group operation for dstSw1
+     * @param dstSw1 one of the paired destination switches
+     * @param edgeLabel2 label to be pushed as part of group operation for dstSw2
+     * @param dstSw2 the other paired destination switch
+     */
+    public DestinationSet(boolean isMplsSet,
+                          int edgeLabel1, DeviceId dstSw1,
+                          int edgeLabel2, DeviceId dstSw2) {
+        this.mplsSet = isMplsSet;
+        if (dstSw1.toString().compareTo(dstSw2.toString()) <= 0) {
+            this.edgeLabel1 = edgeLabel1;
+            this.dstSw1 = dstSw1;
+            this.edgeLabel2 = edgeLabel2;
+            this.dstSw2 = dstSw2;
+        } else {
+            this.edgeLabel1 = edgeLabel2;
+            this.dstSw1 = dstSw2;
+            this.edgeLabel2 = edgeLabel1;
+            this.dstSw2 = dstSw1;
+        }
+    }
+
+    /**
+     * Default constructor for kryo serialization.
+     */
+    public DestinationSet() {
+        this.edgeLabel1 = NOT_ASSIGNED;
+        this.edgeLabel2 = NOT_ASSIGNED;
+        this.mplsSet = true;
+        this.dstSw1 = DeviceId.NONE;
+        this.dstSw2 = DeviceId.NONE;
+    }
+
+    /**
+     * Factory method for DestinationSet hierarchy.
+     *
+     * @param random the expected behavior.
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param dstSw the destination switch
+     * @return the destination set object.
+     */
+    public static DestinationSet destinationSet(boolean random,
+                                          boolean isMplsSet, DeviceId dstSw) {
+        return random ? new RandomDestinationSet(dstSw)
+                      : new DestinationSet(isMplsSet, dstSw);
+    }
+
+    /**
+     * Factory method for DestinationSet hierarchy.
+     *
+     * @param random the expected behavior.
+     * @param isMplsSet indicates if it is a mpls destination set
+     * @param edgeLabel label to be pushed as part of group operation
+     * @param dstSw the destination switch
+     * @return the destination set object
+     */
+    public static DestinationSet destinationSet(boolean random,
+                                          boolean isMplsSet, int edgeLabel,
+                                          DeviceId dstSw) {
+        return random ? new RandomDestinationSet(edgeLabel, dstSw)
+                      : new DestinationSet(isMplsSet, edgeLabel, dstSw);
+    }
+
+    /**
+     * Factory method for DestinationSet hierarchy.
+     *
+     * @param random the expected behavior.
+     * @return the destination set object
+     */
+    public static DestinationSet destinationSet(boolean random) {
+        return random ? new RandomDestinationSet() : new DestinationSet();
+    }
+
+    /**
+     * Gets the label associated with given destination switch.
+     *
+     * @param dstSw the destination switch
+     * @return integer the label associated with the destination switch
+     */
+    public int getEdgeLabel(DeviceId dstSw) {
+        if (dstSw.equals(dstSw1)) {
+            return edgeLabel1;
+        } else if (dstSw.equals(dstSw2)) {
+            return edgeLabel2;
+        }
+        return NOT_ASSIGNED;
+    }
+
+    /**
+     * Gets all the destination switches in this destination set.
+     *
+     * @return a set of destination switch ids
+     */
+    public Set<DeviceId> getDestinationSwitches() {
+        Set<DeviceId> dests = new HashSet<>();
+        dests.add(dstSw1);
+        if (dstSw2 != null) {
+            dests.add(dstSw2);
+        }
+        return dests;
+    }
+
+    /**
+     * Gets the value of mplsSet.
+     *
+     * @return the value of mplsSet
+     */
+    public boolean mplsSet() {
+        return mplsSet;
+    }
+
+    // The list of destination ids and label are used for comparison.
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof DestinationSet)) {
+            return false;
+        }
+        DestinationSet that = (DestinationSet) o;
+        boolean equal = (this.edgeLabel1 == that.edgeLabel1 &&
+                            this.mplsSet == that.mplsSet &&
+                            this.dstSw1.equals(that.dstSw1));
+        if (this.dstSw2 != null && that.dstSw2 == null ||
+                this.dstSw2 == null && that.dstSw2 != null) {
+            return false;
+        }
+        if (this.dstSw2 != null && that.dstSw2 != null) {
+            equal = equal && (this.edgeLabel2 == that.edgeLabel2 &&
+                                this.dstSw2.equals(that.dstSw2));
+        }
+        return equal;
+    }
+
+    // The list of destination ids and label are used for comparison.
+    @Override
+    public int hashCode() {
+        if (dstSw2 == null) {
+            return Objects.hash(mplsSet, edgeLabel1, dstSw1);
+        }
+        return Objects.hash(mplsSet, edgeLabel1, dstSw1, edgeLabel2, dstSw2);
+    }
+
+    @Override
+    public String toString() {
+        ToStringHelper h = toStringHelper(this)
+                                .add("MplsSet", mplsSet)
+                                .add("DstSw1", dstSw1)
+                                .add("Label1", edgeLabel1);
+        if (dstSw2 != null) {
+            h.add("DstSw2", dstSw2)
+             .add("Label2", edgeLabel2);
+        }
+        return h.toString();
+    }
+}
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NeighborSet.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NeighborSet.java
deleted file mode 100644
index 88799f4..0000000
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NeighborSet.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Copyright 2015-present Open Networking Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.onosproject.segmentrouting.grouphandler;
-
-import org.onosproject.net.DeviceId;
-import org.slf4j.Logger;
-
-import java.util.HashSet;
-import java.util.Objects;
-import java.util.Set;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.slf4j.LoggerFactory.getLogger;
-
-/**
- * Representation of a set of neighbor switch dpids along with edge node
- * label and a destination switch. Meant to be used as a lookup-key in a hash-map
- * to retrieve an ECMP-group that hashes packets to a set of ports connecting to
- * the neighbors in this set towards a specific destination switch.
- */
-public class NeighborSet {
-    private final Set<DeviceId> neighbors;
-    private final int edgeLabel;
-    public static final int NO_EDGE_LABEL = -1;
-    private boolean mplsSet;
-    // the destination switch towards which the neighbors are the next-hops.
-    private final DeviceId dstSw;
-    protected static final Logger log = getLogger(NeighborSet.class);
-
-    /**
-     * Constructor with set of neighbors. Edge label is
-     * default to -1.
-     *
-     * @param neighbors set of neighbors representing the next-hops
-     * @param isMplsSet indicates if it is a mpls neighbor set
-     * @param dstSw the destination switch
-     */
-    public NeighborSet(Set<DeviceId> neighbors, boolean isMplsSet, DeviceId dstSw) {
-        checkNotNull(neighbors);
-        this.edgeLabel = NO_EDGE_LABEL;
-        this.neighbors = new HashSet<>();
-        this.neighbors.addAll(neighbors);
-        this.mplsSet = isMplsSet;
-        this.dstSw = dstSw;
-    }
-
-    /**
-     * Constructor with set of neighbors and edge label.
-     *
-     * @param neighbors set of neighbors representing the next-hops
-     * @param isMplsSet indicates if it is a mpls neighbor set
-     * @param edgeLabel label to be pushed as part of group operation
-     * @param dstSw the destination switch
-     */
-    public NeighborSet(Set<DeviceId> neighbors, boolean isMplsSet,
-                       int edgeLabel, DeviceId dstSw) {
-        checkNotNull(neighbors);
-        this.edgeLabel = edgeLabel;
-        this.neighbors = new HashSet<>();
-        this.neighbors.addAll(neighbors);
-        this.mplsSet = isMplsSet;
-        this.dstSw = dstSw;
-    }
-
-    /**
-     * Default constructor for kryo serialization.
-     */
-    public NeighborSet() {
-        this.edgeLabel = NO_EDGE_LABEL;
-        this.neighbors = new HashSet<>();
-        this.mplsSet = true;
-        this.dstSw = DeviceId.NONE;
-    }
-
-    /**
-     * Factory method for NeighborSet hierarchy.
-     *
-     * @param random the expected behavior.
-     * @param neighbors the set of neighbors representing the next-hops
-     * @param isMplsSet indicates if it is a mpls neighbor set
-     * @param dstSw the destination switch
-     * @return the neighbor set object.
-     */
-    public static NeighborSet neighborSet(boolean random, Set<DeviceId> neighbors,
-                                          boolean isMplsSet, DeviceId dstSw) {
-        return random ? new RandomNeighborSet(neighbors, dstSw)
-                      : new NeighborSet(neighbors, isMplsSet, dstSw);
-    }
-
-    /**
-     * Factory method for NeighborSet hierarchy.
-     *
-     * @param random the expected behavior.
-     * @param neighbors the set of neighbors representing the next-hops
-     * @param isMplsSet indicates if it is a mpls neighbor set
-     * @param edgeLabel label to be pushed as part of group operation
-     * @param dstSw the destination switch
-     * @return the neighbor set object
-     */
-    public static NeighborSet neighborSet(boolean random, Set<DeviceId> neighbors,
-                                          boolean isMplsSet, int edgeLabel,
-                                          DeviceId dstSw) {
-        return random ? new RandomNeighborSet(neighbors, edgeLabel, dstSw)
-                      : new NeighborSet(neighbors, isMplsSet, edgeLabel, dstSw);
-    }
-
-    /**
-     * Factory method for NeighborSet hierarchy.
-     *
-     * @param random the expected behavior.
-     * @return the neighbor set object
-     */
-    public static NeighborSet neighborSet(boolean random) {
-        return random ? new RandomNeighborSet() : new NeighborSet();
-    }
-
-    /**
-     * Gets the neighbors part of neighbor set.
-     *
-     * @return set of neighbor identifiers
-     */
-    public Set<DeviceId> getDeviceIds() {
-        return neighbors;
-    }
-
-    /**
-     * Gets the label associated with neighbor set.
-     *
-     * @return integer
-     */
-    public int getEdgeLabel() {
-        return edgeLabel;
-    }
-
-    /**
-     * Gets the destination switch for this neighbor set.
-     *
-     * @return the destination switch id
-     */
-    public DeviceId getDestinationSw() {
-        return dstSw;
-    }
-
-    /**
-     * Gets the first neighbor of the set. The default
-     * implementation assure the first neighbor is the
-     * first of the set. Subclasses can modify this.
-     *
-     * @return the first neighbor of the set
-     */
-    public DeviceId getFirstNeighbor() {
-        return neighbors.isEmpty() ? DeviceId.NONE : neighbors.iterator().next();
-    }
-
-    /**
-     * Gets the value of mplsSet.
-     *
-     * @return the value of mplsSet
-     */
-    public boolean mplsSet() {
-        return mplsSet;
-    }
-
-    // The list of neighbor ids and label are used for comparison.
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (!(o instanceof NeighborSet)) {
-            return false;
-        }
-        NeighborSet that = (NeighborSet) o;
-        return (this.neighbors.containsAll(that.neighbors) &&
-                that.neighbors.containsAll(this.neighbors) &&
-                this.edgeLabel == that.edgeLabel &&
-                this.mplsSet == that.mplsSet &&
-                this.dstSw.equals(that.dstSw));
-    }
-
-    // The list of neighbor ids and label are used for comparison.
-    @Override
-    public int hashCode() {
-        return Objects.hash(neighbors, edgeLabel, mplsSet, dstSw);
-    }
-
-    @Override
-    public String toString() {
-        return toStringHelper(this)
-                .add("Neighbors", neighbors)
-                .add("Label", edgeLabel)
-                .add("MplsSet", mplsSet)
-                .add("DstSw", dstSw)
-                .toString();
-    }
-}
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NextNeighbors.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NextNeighbors.java
new file mode 100644
index 0000000..1a0507b
--- /dev/null
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/NextNeighbors.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2015-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.segmentrouting.grouphandler;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import org.onosproject.net.DeviceId;
+
+public class NextNeighbors {
+    private final Map<DeviceId, Set<DeviceId>> dstNextHops;
+    private final int nextId;
+
+    public NextNeighbors(Map<DeviceId, Set<DeviceId>> dstNextHops, int nextId) {
+        this.dstNextHops = dstNextHops;
+        this.nextId = nextId;
+    }
+
+    public Map<DeviceId, Set<DeviceId>> dstNextHops() {
+        return dstNextHops;
+    }
+
+    public Set<DeviceId> nextHops(DeviceId deviceId) {
+        return dstNextHops.get(deviceId);
+    }
+
+    public int nextId() {
+        return nextId;
+    }
+
+    public boolean containsNextHop(DeviceId nextHopId) {
+        for (Set<DeviceId> nextHops : dstNextHops.values()) {
+            if (nextHops != null && nextHops.contains(nextHopId)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public Set<DeviceId> getDstForNextHop(DeviceId nextHopId) {
+        Set<DeviceId> dstSet = new HashSet<>();
+        for (DeviceId dstKey : dstNextHops.keySet()) {
+            if (dstNextHops.get(dstKey).contains(nextHopId)) {
+                dstSet.add(dstKey);
+            }
+        }
+        return dstSet;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof NextNeighbors)) {
+            return false;
+        }
+        NextNeighbors that = (NextNeighbors) o;
+        return (this.nextId == that.nextId) &&
+                this.dstNextHops.equals(that.dstNextHops);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nextId, dstNextHops);
+    }
+
+    @Override
+    public String toString() {
+        return toStringHelper(this)
+                .add("nextId", nextId)
+                .add("dstNextHops", dstNextHops)
+                .toString();
+    }
+}
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
index 2ba7c19..c0935ce 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
@@ -199,7 +199,7 @@
                     tBuilder.setOutput(bucketId.outPort())
                             .setEthDst(neighborEthDst)
                             .setEthSrc(nodeMacAddr);
-                    if (bucketId.label() != NeighborSet.NO_EDGE_LABEL) {
+                    if (bucketId.label() != DestinationSet.NO_EDGE_LABEL) {
                         tBuilder.pushMpls()
                             .setMpls(MplsLabel.mplsLabel(bucketId.label()));
                     }
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomNeighborSet.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomDestinationSet.java
similarity index 66%
rename from apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomNeighborSet.java
rename to apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomDestinationSet.java
index edf178f..3671b03 100644
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomNeighborSet.java
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/grouphandler/RandomDestinationSet.java
@@ -16,32 +16,32 @@
 
 package org.onosproject.segmentrouting.grouphandler;
 
-import com.google.common.collect.Iterables;
-import org.apache.commons.lang3.RandomUtils;
 import org.onosproject.net.DeviceId;
 
-import java.util.Set;
-
 /**
  * Extends its super class modifying its internal behavior.
  * Pick a neighbor will pick a random neighbor.
  */
-public class RandomNeighborSet extends NeighborSet {
+public class RandomDestinationSet extends DestinationSet {
 
-    public RandomNeighborSet(Set<DeviceId> neighbors, DeviceId dstSw) {
-        super(neighbors, true, dstSw);
+    public RandomDestinationSet(DeviceId dstSw) {
+        super(true, dstSw);
     }
 
-    public RandomNeighborSet(Set<DeviceId> neighbors, int edgeLabel,
+    public RandomDestinationSet(int edgeLabel,
                              DeviceId dstSw) {
-        super(neighbors, true, edgeLabel, dstSw);
+        super(true, edgeLabel, dstSw);
     }
 
-    public RandomNeighborSet() {
+    public RandomDestinationSet() {
         super();
     }
 
-    @Override
+    // XXX revisit the need for this class since neighbors no longer stored here
+    // will be handled when we fix pseudowires for dual-Tor scenarios
+
+
+    /*@Override
     public DeviceId getFirstNeighbor() {
         if (getDeviceIds().isEmpty()) {
             return DeviceId.NONE;
@@ -49,12 +49,12 @@
         int size = getDeviceIds().size();
         int index = RandomUtils.nextInt(0, size);
         return Iterables.get(getDeviceIds(), index);
-    }
+    }*/
 
     @Override
     public String toString() {
-        return " RandomNeighborset Sw: " + getDeviceIds()
-                + " and Label: " + getEdgeLabel()
-                + " for destination: " + getDestinationSw();
+        return " RandomNeighborset Sw: " //+ getDeviceIds()
+                + " and Label: " //+ getEdgeLabel()
+                + " for destination: "; // + getDestinationSw();
     }
 }
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/storekey/DestinationSetNextObjectiveStoreKey.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/storekey/DestinationSetNextObjectiveStoreKey.java
new file mode 100644
index 0000000..9b6e621
--- /dev/null
+++ b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/storekey/DestinationSetNextObjectiveStoreKey.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2016-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licedses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.segmentrouting.storekey;
+
+import java.util.Objects;
+
+import org.onosproject.net.DeviceId;
+import org.onosproject.segmentrouting.grouphandler.DestinationSet;
+
+/**
+ * Key of Destination set next objective store.
+ */
+public class DestinationSetNextObjectiveStoreKey {
+    private final DeviceId deviceId;
+    private final DestinationSet ds;
+
+    /**
+     * Constructs the key of destination set next objective store.
+     *
+     * @param deviceId device ID
+     * @param ds destination set
+     */
+    public DestinationSetNextObjectiveStoreKey(DeviceId deviceId,
+                                            DestinationSet ds) {
+        this.deviceId = deviceId;
+        this.ds = ds;
+    }
+
+    /**
+     * Returns the device ID in the key.
+     *
+     * @return device ID
+     */
+    public DeviceId deviceId() {
+        return this.deviceId;
+    }
+
+    /**
+     * Returns the destination set in the key.
+     *
+     * @return destination set
+     */
+    public DestinationSet destinationSet() {
+        return this.ds;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof DestinationSetNextObjectiveStoreKey)) {
+            return false;
+        }
+        DestinationSetNextObjectiveStoreKey that =
+                (DestinationSetNextObjectiveStoreKey) o;
+        return (Objects.equals(this.deviceId, that.deviceId) &&
+                Objects.equals(this.ds, that.ds));
+    }
+
+    // The list of destination ids and label are used for comparison.
+    @Override
+    public int hashCode() {
+        int result = 17;
+        result = 31 * result + Objects.hashCode(this.deviceId)
+                + Objects.hashCode(this.ds);
+
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "Device: " + deviceId + " " + ds;
+    }
+}
diff --git a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/storekey/NeighborSetNextObjectiveStoreKey.java b/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/storekey/NeighborSetNextObjectiveStoreKey.java
deleted file mode 100644
index b9ce6a9..0000000
--- a/apps/segmentrouting/src/main/java/org/onosproject/segmentrouting/storekey/NeighborSetNextObjectiveStoreKey.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright 2016-present Open Networking Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.onosproject.segmentrouting.storekey;
-
-import java.util.Objects;
-
-import org.onosproject.net.DeviceId;
-import org.onosproject.segmentrouting.grouphandler.NeighborSet;
-
-/**
- * Key of Neighborset next objective store.
- */
-public class NeighborSetNextObjectiveStoreKey {
-    private final DeviceId deviceId;
-    private final NeighborSet ns;
-
-    /**
-     * Constructs the key of neighbor set next objective store.
-     *
-     * @param deviceId device ID
-     * @param ns neighbor set
-     */
-    public NeighborSetNextObjectiveStoreKey(DeviceId deviceId,
-                                            NeighborSet ns) {
-        this.deviceId = deviceId;
-        this.ns = ns;
-    }
-
-    /**
-     * Returns the device ID in the key.
-     *
-     * @return device ID
-     */
-    public DeviceId deviceId() {
-        return this.deviceId;
-    }
-
-    /**
-     * Returns the neighbor set in the key.
-     *
-     * @return neighbor set
-     */
-    public NeighborSet neighborSet() {
-        return this.ns;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (!(o instanceof NeighborSetNextObjectiveStoreKey)) {
-            return false;
-        }
-        NeighborSetNextObjectiveStoreKey that =
-                (NeighborSetNextObjectiveStoreKey) o;
-        return (Objects.equals(this.deviceId, that.deviceId) &&
-                Objects.equals(this.ns, that.ns));
-    }
-
-    // The list of neighbor ids and label are used for comparison.
-    @Override
-    public int hashCode() {
-        int result = 17;
-        result = 31 * result + Objects.hashCode(this.deviceId)
-                + Objects.hashCode(this.ns);
-
-        return result;
-    }
-
-    @Override
-    public String toString() {
-        return "Device: " + deviceId + " " + ns;
-    }
-}
diff --git a/apps/segmentrouting/src/main/resources/OSGI-INF/blueprint/shell-config.xml b/apps/segmentrouting/src/main/resources/OSGI-INF/blueprint/shell-config.xml
index 97a9425..3003a30 100644
--- a/apps/segmentrouting/src/main/resources/OSGI-INF/blueprint/shell-config.xml
+++ b/apps/segmentrouting/src/main/resources/OSGI-INF/blueprint/shell-config.xml
@@ -46,7 +46,7 @@
             <action class="org.onosproject.segmentrouting.cli.EcmpGraphCommand"/>
         </command>
         <command>
-            <action class="org.onosproject.segmentrouting.cli.NeighborSetCommand"/>
+            <action class="org.onosproject.segmentrouting.cli.NextHopCommand"/>
         </command>
     </command-bundle>
 </blueprint>
diff --git a/apps/segmentrouting/src/test/java/org/onosproject/segmentrouting/grouphandler/DestinationSetTest.java b/apps/segmentrouting/src/test/java/org/onosproject/segmentrouting/grouphandler/DestinationSetTest.java
new file mode 100644
index 0000000..f890852
--- /dev/null
+++ b/apps/segmentrouting/src/test/java/org/onosproject/segmentrouting/grouphandler/DestinationSetTest.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.segmentrouting.grouphandler;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.onosproject.net.DeviceId;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class DestinationSetTest {
+    DestinationSet ds1, ds2, ds3, ds4;
+    DeviceId d201, d202;
+    int el201, el202;
+
+    @Before
+    public void setUp() {
+        d201 = DeviceId.deviceId("of:0000000000000201");
+        d202 = DeviceId.deviceId("of:0000000000000202");
+        el201 = 201;
+        el202 = 202;
+        ds1 = new DestinationSet(false, d201);
+        ds2 = new DestinationSet(false, el201, d201);
+        ds3 = new DestinationSet(false, el201, d201, el202, d202);
+        ds4 = new DestinationSet(false,
+                                 DestinationSet.NO_EDGE_LABEL, d201,
+                                 DestinationSet.NO_EDGE_LABEL, d202);
+    }
+
+    @Test
+    public void testIsValid() {
+        assertTrue(!ds1.mplsSet());
+        assertTrue(ds1.getEdgeLabel(d201) == DestinationSet.NO_EDGE_LABEL);
+        assertTrue(ds1.getDestinationSwitches().size() == 1);
+
+        assertTrue(!ds2.mplsSet());
+        assertTrue(ds2.getEdgeLabel(d201) == el201);
+        assertTrue(ds2.getDestinationSwitches().size() == 1);
+
+        assertTrue(!ds3.mplsSet());
+        assertTrue(ds3.getEdgeLabel(d201) == el201);
+        assertTrue(ds3.getEdgeLabel(d202) == el202);
+        assertTrue(ds3.getDestinationSwitches().size() == 2);
+
+        assertTrue(!ds4.mplsSet());
+        assertTrue(ds4.getEdgeLabel(d201) == DestinationSet.NO_EDGE_LABEL);
+        assertTrue(ds4.getEdgeLabel(d202) == DestinationSet.NO_EDGE_LABEL);
+        assertTrue(ds4.getDestinationSwitches().size() == 2);
+
+        assertFalse(ds1.equals(ds2));
+        assertFalse(ds1.equals(ds4));
+        assertFalse(ds3.equals(ds4));
+        assertFalse(ds2.equals(ds3));
+    }
+
+
+    @Test
+    public void testOneDestinationWithoutLabel() {
+        DestinationSet testds = new DestinationSet(false, d201);
+        assertTrue(testds.equals(ds1)); // match
+
+        testds = new DestinationSet(true, d201);
+        assertFalse(testds.equals(ds1)); //wrong mplsSet
+
+        testds = new DestinationSet(false, d202);
+        assertFalse(testds.equals(ds1)); //wrong device
+
+        testds = new DestinationSet(false, el201, d201);
+        assertFalse(testds.equals(ds1)); // wrong label
+
+        testds = new DestinationSet(false, -1, d201, -1, d202);
+        assertFalse(testds.equals(ds1)); // 2-devs should not match
+    }
+
+
+
+    @Test
+    public void testOneDestinationWithLabel() {
+        DestinationSet testds = new DestinationSet(false, 203, d202);
+        assertFalse(testds.equals(ds2)); //wrong label
+
+        testds = new DestinationSet(true, 201, d201);
+        assertFalse(testds.equals(ds2)); //wrong mplsSet
+
+        testds = new DestinationSet(false, 201, d202);
+        assertFalse(testds.equals(ds2)); //wrong device
+
+        testds = new DestinationSet(false, 201, DeviceId.deviceId("of:0000000000000201"));
+        assertTrue(testds.equals(ds2)); // match
+
+        testds = new DestinationSet(false, d201);
+        assertFalse(testds.equals(ds2)); // wrong label
+
+        testds = new DestinationSet(false, el201, d201, el202, d202);
+        assertFalse(testds.equals(ds1)); // 2-devs should not match
+    }
+
+    @Test
+    public void testDestPairWithLabel() {
+        DestinationSet testds = new DestinationSet(false, el201, d201, el202, d202);
+        assertTrue(testds.equals(ds3)); // match same switches, same order
+        assertTrue(testds.hashCode() == ds3.hashCode());
+
+        testds = new DestinationSet(false, el202, d202, el201, d201);
+        assertTrue(testds.equals(ds3)); // match same switches, order reversed
+        assertTrue(testds.hashCode() == ds3.hashCode());
+
+        testds = new DestinationSet(false, el202, d202);
+        assertFalse(testds.equals(ds3)); // one less switch should not match
+        assertFalse(testds.hashCode() == ds3.hashCode());
+
+        testds = new DestinationSet(false, el201, d201);
+        assertFalse(testds.equals(ds3)); // one less switch should not match
+        assertFalse(testds.hashCode() == ds3.hashCode());
+
+        testds = new DestinationSet(false, el201, d201, 0, DeviceId.NONE);
+        assertFalse(testds.equals(ds3)); // one less switch should not match
+        assertFalse(testds.hashCode() == ds3.hashCode());
+
+        testds = new DestinationSet(false, el201, d202, el201, d201);
+        assertFalse(testds.equals(ds3)); // wrong labels
+        assertFalse(testds.hashCode() == ds3.hashCode());
+
+        testds = new DestinationSet(true, el202, d202, el201, d201);
+        assertFalse(testds.equals(ds3)); // wrong mpls set
+        assertFalse(testds.hashCode() == ds3.hashCode());
+
+        testds = new DestinationSet(false, el202, d202, el201, d202);
+        assertFalse(testds.equals(ds3)); // wrong device
+        assertFalse(testds.hashCode() == ds3.hashCode());
+
+        testds = new DestinationSet(false,
+                                    el202, DeviceId.deviceId("of:0000000000000205"),
+                                    el201, d201);
+        assertFalse(testds.equals(ds3)); // wrong device
+        assertFalse(testds.hashCode() == ds3.hashCode());
+    }
+
+    @Test
+    public void testDestPairWithoutLabel() {
+        DestinationSet testds = new DestinationSet(false, -1, d201, -1, d202);
+        assertTrue(testds.equals(ds4)); // match same switches, same order
+        assertTrue(testds.hashCode() == ds4.hashCode());
+
+        testds = new DestinationSet(false, -1, d202, -1, d201);
+        assertTrue(testds.equals(ds4)); // match same switches, order reversed
+        assertTrue(testds.hashCode() == ds4.hashCode());
+
+        testds = new DestinationSet(false, -1, d202);
+        assertFalse(testds.equals(ds4)); // one less switch should not match
+        assertFalse(testds.hashCode() == ds4.hashCode());
+
+        testds = new DestinationSet(false, -1, d201);
+        assertFalse(testds.equals(ds4)); // one less switch should not match
+        assertFalse(testds.hashCode() == ds4.hashCode());
+
+        testds = new DestinationSet(false, -1, d201, 0, DeviceId.NONE);
+        assertFalse(testds.equals(ds4)); // one less switch should not match
+        assertFalse(testds.hashCode() == ds4.hashCode());
+
+        testds = new DestinationSet(false, el201, d201, -1, d202);
+        assertFalse(testds.equals(ds4)); // wrong labels
+        assertFalse(testds.hashCode() == ds4.hashCode());
+
+        testds = new DestinationSet(true, -1, d202, -1, d201);
+        assertFalse(testds.equals(ds4)); // wrong mpls set
+        assertFalse(testds.hashCode() == ds4.hashCode());
+
+        testds = new DestinationSet(false, -1, d202, -1, d202);
+        assertFalse(testds.equals(ds4)); // wrong device
+        assertFalse(testds.hashCode() == ds4.hashCode());
+
+        testds = new DestinationSet(false,
+                                    -1, DeviceId.deviceId("of:0000000000000205"),
+                                    -1, d201);
+        assertFalse(testds.equals(ds4)); // wrong device
+        assertFalse(testds.hashCode() == ds4.hashCode());
+    }
+
+}