ONOS-1823 and ONOS-1838:Segment Routing Multi-instance Support-1

Change-Id: I3cc848415a609a9c4001d135e51104c62fb2830d
diff --git a/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java b/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
index 0fa2aca..14187c0 100644
--- a/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
@@ -89,14 +89,18 @@
         populationStatus = Status.STARTED;
         rulePopulator.resetCounter();
         log.info("Starts to populate routing rules");
+        log.debug("populateAllRoutingRules: populationStatus is STARTED");
 
         for (Device sw : srManager.deviceService.getDevices()) {
             if (srManager.mastershipService.getLocalRole(sw.id()) != MastershipRole.MASTER) {
+                log.debug("populateAllRoutingRules: skipping device {}...we are not master",
+                          sw.id());
                 continue;
             }
 
             ECMPShortestPathGraph ecmpSpg = new ECMPShortestPathGraph(sw.id(), srManager);
             if (!populateEcmpRoutingRules(sw.id(), ecmpSpg)) {
+                log.debug("populateAllRoutingRules: populationStatus is ABORTED");
                 populationStatus = Status.ABORTED;
                 log.debug("Abort routing rule population");
                 return false;
@@ -106,6 +110,7 @@
             // TODO: Set adjacency routing rule for all switches
         }
 
+        log.debug("populateAllRoutingRules: populationStatus is SUCCEEDED");
         populationStatus = Status.SUCCEEDED;
         log.info("Completes routing rule population. Total # of rules pushed : {}",
                 rulePopulator.getCounter());
@@ -144,6 +149,8 @@
             log.info("Starts rule population from link change");
 
             Set<ArrayList<DeviceId>> routeChanges;
+            log.trace("populateRoutingRulesForLinkStatusChange: "
+                    + "populationStatus is STARTED");
             populationStatus = Status.STARTED;
             if (linkFail == null) {
                 // Compare all routes of existing ECMP SPG with the new ones
@@ -155,16 +162,19 @@
 
             if (routeChanges.isEmpty()) {
                 log.info("No route changes for the link status change");
+                log.debug("populateRoutingRulesForLinkStatusChange: populationStatus is SUCCEEDED");
                 populationStatus = Status.SUCCEEDED;
                 return true;
             }
 
             if (repopulateRoutingRulesForRoutes(routeChanges)) {
+                log.debug("populateRoutingRulesForLinkStatusChange: populationStatus is SUCCEEDED");
                 populationStatus = Status.SUCCEEDED;
                 log.info("Complete to repopulate the rules. # of rules populated : {}",
                         rulePopulator.getCounter());
                 return true;
             } else {
+                log.debug("populateRoutingRulesForLinkStatusChange: populationStatus is ABORTED");
                 populationStatus = Status.ABORTED;
                 log.warn("Failed to repopulate the rules.");
                 return false;
@@ -177,6 +187,7 @@
         for (ArrayList<DeviceId> link: routes) {
             // When only the source device is defined, reinstall routes to all other devices
             if (link.size() == 1) {
+                log.trace("repopulateRoutingRulesForRoutes: running ECMP graph for device {}", link.get(0));
                 ECMPShortestPathGraph ecmpSpg = new ECMPShortestPathGraph(link.get(0), srManager);
                 if (populateEcmpRoutingRules(link.get(0), ecmpSpg)) {
                     currentEcmpSpgMap.put(link.get(0), ecmpSpg);
@@ -187,8 +198,7 @@
             } else {
                 DeviceId src = link.get(0);
                 DeviceId dst = link.get(1);
-                log.trace("repopulateRoutingRulesForRoutes: running ECMP graph "
-                        + "for device {}", dst);
+                log.trace("repopulateRoutingRulesForRoutes: running ECMP graph for device {}", dst);
                 ECMPShortestPathGraph ecmpSpg = updatedEcmpSpgMap.get(dst);
                 HashMap<Integer, HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>>> switchVia =
                         ecmpSpg.getAllLearnedSwitchesAndVia();
@@ -278,14 +288,12 @@
             log.debug("Checking route change for switch {}", sw.id());
             ECMPShortestPathGraph ecmpSpg = currentEcmpSpgMap.get(sw.id());
             if (ecmpSpg == null) {
-                log.debug("No existing ECMP path for Switch {}", sw.id());
+                log.debug("No existing ECMP graph for device {}", sw.id());
                 ArrayList<DeviceId> route = new ArrayList<>();
                 route.add(sw.id());
                 routes.add(route);
                 continue;
             }
-            log.debug("computeRouteChange: running ECMP graph "
-                    + "for device {}", sw.id());
             ECMPShortestPathGraph newEcmpSpg = updatedEcmpSpgMap.get(sw.id());
             currentEcmpSpgMap.put(sw.id(), newEcmpSpg);
             HashMap<Integer, HashMap<DeviceId, ArrayList<ArrayList<DeviceId>>>> switchVia =
@@ -400,6 +408,8 @@
         // rule for both subnet and router IP.
         if (config.isEdgeDevice(targetSw) && config.isEdgeDevice(destSw)) {
             List<Ip4Prefix> subnets = config.getSubnets(destSw);
+            log.debug("populateEcmpRoutingRulePartial in device {} towards {} for subnets {}",
+                    targetSw, destSw, subnets);
             result = rulePopulator.populateIpRuleForSubnet(targetSw,
                                                            subnets,
                                                            destSw,
@@ -410,6 +420,8 @@
 
             Ip4Address routerIp = config.getRouterIp(destSw);
             IpPrefix routerIpPrefix = IpPrefix.valueOf(routerIp, IpPrefix.MAX_INET_MASK_LENGTH);
+            log.debug("populateEcmpRoutingRulePartial in device {} towards {} for router IP {}",
+                    targetSw, destSw, routerIpPrefix);
             result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix, destSw, nextHops);
             if (!result) {
                 return false;
@@ -419,6 +431,8 @@
         } else if (config.isEdgeDevice(targetSw)) {
             Ip4Address routerIp = config.getRouterIp(destSw);
             IpPrefix routerIpPrefix = IpPrefix.valueOf(routerIp, IpPrefix.MAX_INET_MASK_LENGTH);
+            log.debug("populateEcmpRoutingRulePartial in device {} towards {} for router IP {}",
+                    targetSw, destSw, routerIpPrefix);
             result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix, destSw, nextHops);
             if (!result) {
                 return false;
@@ -426,6 +440,8 @@
         }
 
         // Populates MPLS rules to all routers
+        log.debug("populateEcmpRoutingRulePartial in device{} towards {} for all MPLS rules",
+                targetSw, destSw);
         result = rulePopulator.populateMplsRule(targetSw, destSw, nextHops);
         if (!result) {
             return false;
@@ -453,9 +469,13 @@
     public void startPopulationProcess() {
         synchronized (populationStatus) {
             if (populationStatus == Status.IDLE
-                    || populationStatus == Status.SUCCEEDED) {
+                    || populationStatus == Status.SUCCEEDED
+                    || populationStatus == Status.ABORTED) {
                 populationStatus = Status.STARTED;
                 populateAllRoutingRules();
+            } else {
+                log.warn("Not initiating startPopulationProcess as populationStatus is {}",
+                         populationStatus);
             }
         }
     }
diff --git a/src/main/java/org/onosproject/segmentrouting/DeviceConfiguration.java b/src/main/java/org/onosproject/segmentrouting/DeviceConfiguration.java
index d0721e9..ad8c0a3 100644
--- a/src/main/java/org/onosproject/segmentrouting/DeviceConfiguration.java
+++ b/src/main/java/org/onosproject/segmentrouting/DeviceConfiguration.java
@@ -99,6 +99,9 @@
                     deviceConfigMap.get(deviceId).nodeSid);
             return deviceConfigMap.get(deviceId).nodeSid;
         } else {
+            log.warn("getSegmentId for device {} "
+                    + "throwing IllegalStateException "
+                    + "because device does not exist in config", deviceId);
             throw new IllegalStateException();
         }
     }
@@ -151,6 +154,9 @@
                     deviceConfigMap.get(deviceId).mac);
             return deviceConfigMap.get(deviceId).mac;
         } else {
+            log.warn("getDeviceMac for device {} "
+                    + "throwing IllegalStateException "
+                    + "because device does not exist in config", deviceId);
             throw new IllegalStateException();
         }
     }
@@ -168,6 +174,9 @@
                     deviceConfigMap.get(deviceId).ip);
             return deviceConfigMap.get(deviceId).ip;
         } else {
+            log.warn("getRouterIp for device {} "
+                    + "throwing IllegalStateException "
+                    + "because device does not exist in config", deviceId);
             throw new IllegalStateException();
         }
     }
@@ -187,6 +196,9 @@
                     deviceConfigMap.get(deviceId).isEdge);
             return deviceConfigMap.get(deviceId).isEdge;
         } else {
+            log.warn("isEdgeDevice for device {} "
+                    + "throwing IllegalStateException "
+                    + "because device does not exist in config", deviceId);
             throw new IllegalStateException();
         }
     }
diff --git a/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java b/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
index f3ccaa0..7092093 100644
--- a/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
+++ b/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
@@ -217,6 +217,11 @@
 
         // If the next hop is the destination router, do PHP
         if (nextHops.size() == 1 && destSwId.equals(nextHops.toArray()[0])) {
+            log.debug("populateMplsRule: Installing MPLS forwarding objective for "
+                    + "label {} in switch {} with PHP",
+                    config.getSegmentId(destSwId),
+                    deviceId);
+
             ForwardingObjective.Builder fwdObjBosBuilder =
                     getMplsForwardingObjective(deviceId,
                                                destSwId,
@@ -237,6 +242,11 @@
                 return false;
             }
         } else {
+            log.debug("Installing MPLS forwarding objective for "
+                    + "label {} in switch {} without PHP",
+                    config.getSegmentId(destSwId),
+                    deviceId);
+
             ForwardingObjective.Builder fwdObjBosBuilder =
                     getMplsForwardingObjective(deviceId,
                                                destSwId,
@@ -264,8 +274,6 @@
                     .makePermanent()).withSelector(selector)
                     .withPriority(100))
                     .withFlag(ForwardingObjective.Flag.SPECIFIC);
-            log.debug("Installing MPLS forwarding objective in switch {}",
-                    deviceId);
             srManager.flowObjectiveService.forward(deviceId,
                                                    fwdObjBuilder.add());
             rulePopulationCounter.incrementAndGet();
diff --git a/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java b/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
index 9ace6b8..87282b4 100644
--- a/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
+++ b/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
@@ -22,23 +22,22 @@
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.IPv4;
+import org.onlab.util.KryoNamespace;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.CoreService;
 import org.onosproject.event.Event;
 import org.onosproject.segmentrouting.grouphandler.DefaultGroupHandler;
 import org.onosproject.segmentrouting.grouphandler.NeighborSet;
+import org.onosproject.segmentrouting.grouphandler.NeighborSetNextObjectiveStoreKey;
 import org.onosproject.mastership.MastershipService;
 import org.onosproject.net.Device;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Link;
-import org.onosproject.net.MastershipRole;
 import org.onosproject.net.Port;
 import org.onosproject.net.device.DeviceEvent;
 import org.onosproject.net.device.DeviceListener;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
-import org.onosproject.net.group.Group;
-import org.onosproject.net.group.GroupEvent;
 import org.onosproject.net.group.GroupKey;
 import org.onosproject.net.host.HostService;
 import org.onosproject.net.intent.IntentService;
@@ -51,9 +50,16 @@
 import org.onosproject.net.packet.PacketService;
 import org.onosproject.net.topology.TopologyService;
 import org.onosproject.segmentrouting.config.NetworkConfigManager;
+import org.onosproject.store.service.EventuallyConsistentMap;
+import org.onosproject.store.service.EventuallyConsistentMapBuilder;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.WallClockTimestamp;
+import org.onosproject.store.service.WallclockClockManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.URI;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -112,6 +118,11 @@
     private static ScheduledFuture<?> eventHandlerFuture = null;
     private ConcurrentLinkedQueue<Event> eventQueue = new ConcurrentLinkedQueue<Event>();
     private Map<DeviceId, DefaultGroupHandler> groupHandlerMap = new ConcurrentHashMap<DeviceId, DefaultGroupHandler>();
+    // Per device next objective ID store with (device id + neighbor set) as key
+    private EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey,
+        Integer> nsNextObjStore = null;
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected StorageService storageService;
 
     private NetworkConfigManager networkConfigService = new NetworkConfigManager();;
 
@@ -119,10 +130,34 @@
     private static int numOfHandlerExecution = 0;
     private static int numOfHandlerScheduled = 0;
 
+    private KryoNamespace.Builder kryoBuilder = null;
+
     @Activate
     protected void activate() {
         appId = coreService
                 .registerApplication("org.onosproject.segmentrouting");
+
+        kryoBuilder = new KryoNamespace.Builder()
+            .register(NeighborSetNextObjectiveStoreKey.class,
+                  NeighborSet.class,
+                  DeviceId.class,
+                  URI.class,
+                  WallClockTimestamp.class,
+                  org.onosproject.cluster.NodeId.class,
+                  HashSet.class
+                );
+
+        log.debug("Creating EC map nsnextobjectivestore");
+        EventuallyConsistentMapBuilder<NeighborSetNextObjectiveStoreKey, Integer>
+                nsNextObjMapBuilder = storageService.eventuallyConsistentMapBuilder();
+
+        nsNextObjStore = nsNextObjMapBuilder
+                .withName("nsnextobjectivestore")
+                .withSerializer(kryoBuilder)
+                .withClockService(new WallclockClockManager<>())
+                .build();
+        log.trace("Current size {}", nsNextObjStore.size());
+
         networkConfigService.init();
         deviceConfiguration = new DeviceConfiguration(networkConfigService);
         arpHandler = new ArpHandler(this);
@@ -136,20 +171,18 @@
         deviceService.addListener(new InternalDeviceListener());
 
         for (Device device : deviceService.getDevices()) {
-            if (mastershipService.getLocalRole(device.id()) == MastershipRole.MASTER) {
-                DefaultGroupHandler groupHandler = DefaultGroupHandler
-                        .createGroupHandler(device.id(), appId,
-                                            deviceConfiguration, linkService,
-                                            flowObjectiveService);
-                groupHandlerMap.put(device.id(), groupHandler);
-                defaultRoutingHandler.populateTtpRules(device.id());
-                log.debug("Initiating default group handling for {}", device.id());
-            } else {
-                log.debug("Activate: Local role {} "
-                                  + "is not MASTER for device {}",
-                          mastershipService.getLocalRole(device.id()),
-                          device.id());
-            }
+            //Irrespective whether the local is a MASTER or not for this device,
+            //create group handler instance and push default TTP flow rules.
+            //Because in a multi-instance setup, instances can initiate
+            //groups for any devices. Also the default TTP rules are needed
+            //to be pushed before inserting any IP table entries for any device
+            DefaultGroupHandler groupHandler = DefaultGroupHandler
+                    .createGroupHandler(device.id(), appId,
+                                        deviceConfiguration, linkService,
+                                        flowObjectiveService,
+                                        nsNextObjStore);
+            groupHandlerMap.put(device.id(), groupHandler);
+            defaultRoutingHandler.populateTtpRules(device.id());
         }
 
         defaultRoutingHandler.startPopulationProcess();
@@ -180,8 +213,14 @@
 
     public int getNextObjectiveId(DeviceId deviceId, NeighborSet ns) {
 
-        return (groupHandlerMap.get(deviceId) != null) ? groupHandlerMap
-                .get(deviceId).getNextObjectiveId(ns) : -1;
+        if (groupHandlerMap.get(deviceId) != null) {
+            log.trace("getNextObjectiveId query in device {}", deviceId);
+            return groupHandlerMap
+                    .get(deviceId).getNextObjectiveId(ns);
+        } else {
+            log.warn("getNextObjectiveId query in device {} not found", deviceId);
+            return -1;
+        }
     }
 
     private class InternalPacketProcessor implements PacketProcessor {
@@ -224,12 +263,12 @@
 
         @Override
         public void event(DeviceEvent event) {
-            if (mastershipService.getLocalRole(event.subject().id()) != MastershipRole.MASTER) {
+            /*if (mastershipService.getLocalRole(event.subject().id()) != MastershipRole.MASTER) {
                 log.debug("Local role {} is not MASTER for device {}",
                           mastershipService.getLocalRole(event.subject().id()),
                           event.subject().id());
                 return;
-            }
+            }*/
 
             switch (event.type()) {
             case DEVICE_ADDED:
@@ -245,12 +284,14 @@
 
     private void scheduleEventHandlerIfNotScheduled(Event event) {
 
-        eventQueue.add(event);
-        numOfEvents++;
-        if (eventHandlerFuture == null || eventHandlerFuture.isDone()) {
-            eventHandlerFuture = executorService
-                    .schedule(eventHandler, 100, TimeUnit.MILLISECONDS);
-            numOfHandlerScheduled++;
+        synchronized (eventQueue) {
+            eventQueue.add(event);
+            numOfEvents++;
+            if (eventHandlerFuture == null || eventHandlerFuture.isDone()) {
+                eventHandlerFuture = executorService
+                        .schedule(eventHandler, 100, TimeUnit.MILLISECONDS);
+                numOfHandlerScheduled++;
+            }
         }
 
         log.trace("numOfEvents {}, numOfEventHanlderScheduled {}", numOfEvents,
@@ -262,44 +303,68 @@
 
         @Override
         public void run() {
-            numOfHandlerExecution++;
-            while (!eventQueue.isEmpty()) {
-                Event event = eventQueue.poll();
-                if (event.type() == LinkEvent.Type.LINK_ADDED) {
-                    processLinkAdded((Link) event.subject());
-                } else if (event.type() == LinkEvent.Type.LINK_REMOVED) {
-                    processLinkRemoved((Link) event.subject());
-                } else if (event.type() == GroupEvent.Type.GROUP_ADDED) {
-                    processGroupAdded((Group) event.subject());
-                } else if (event.type() == DeviceEvent.Type.DEVICE_ADDED ||
-                        event.type() == DeviceEvent.Type.DEVICE_AVAILABILITY_CHANGED ||
-                        event.type() == DeviceEvent.Type.DEVICE_UPDATED) {
-                    if (deviceService.isAvailable(((Device) event.subject()).id())) {
-                        processDeviceAdded((Device) event.subject());
+            try {
+                synchronized (eventQueue) {
+                    numOfHandlerExecution++;
+                    while (!eventQueue.isEmpty()) {
+                        Event event = eventQueue.poll();
+                        if (event.type() == LinkEvent.Type.LINK_ADDED) {
+                            processLinkAdded((Link) event.subject());
+                        } else if (event.type() == LinkEvent.Type.LINK_REMOVED) {
+                            processLinkRemoved((Link) event.subject());
+                        //} else if (event.type() == GroupEvent.Type.GROUP_ADDED) {
+                        //    processGroupAdded((Group) event.subject());
+                        } else if (event.type() == DeviceEvent.Type.DEVICE_ADDED ||
+                                event.type() == DeviceEvent.Type.DEVICE_AVAILABILITY_CHANGED ||
+                                event.type() == DeviceEvent.Type.DEVICE_UPDATED) {
+                            if (deviceService.isAvailable(((Device) event.subject()).id())) {
+                                processDeviceAdded((Device) event.subject());
+                            }
+                        } else if (event.type() == DeviceEvent.Type.PORT_REMOVED) {
+                            processPortRemoved((Device) event.subject(),
+                                               ((DeviceEvent) event).port());
+                        } else {
+                            log.warn("Unhandled event type: {}", event.type());
+                        }
                     }
-                } else if (event.type() == DeviceEvent.Type.PORT_REMOVED) {
-                    processPortRemoved((Device) event.subject(),
-                                       ((DeviceEvent) event).port());
-                } else {
-                    log.warn("Unhandled event type: {}", event.type());
                 }
+                log.debug("numOfHandlerExecution {} numOfEventHanlderScheduled {} numOfEvents {}",
+                          numOfHandlerExecution, numOfHandlerScheduled, numOfEvents);
+            } catch (Exception e) {
+                log.error("SegmentRouting event handler "
+                        + "thread thrown an exception: {}", e);
             }
-            log.debug("numOfHandlerExecution {} numOfEventHanlderScheduled {} numOfEvents {}",
-                      numOfHandlerExecution, numOfHandlerScheduled, numOfEvents);
         }
     }
 
     private void processLinkAdded(Link link) {
         log.debug("A new link {} was added", link.toString());
 
-        if (mastershipService.getLocalRole(link.src().deviceId()) == MastershipRole.MASTER) {
-            DefaultGroupHandler groupHandler = groupHandlerMap.get(link.src()
-                    .deviceId());
-            if (groupHandler != null) {
+        //Irrespective whether the local is a MASTER or not for this device,
+        //create group handler instance and push default TTP flow rules.
+        //Because in a multi-instance setup, instances can initiate
+        //groups for any devices. Also the default TTP rules are needed
+        //to be pushed before inserting any IP table entries for any device
+        DefaultGroupHandler groupHandler = groupHandlerMap.get(link.src()
+                .deviceId());
+        if (groupHandler != null) {
+            groupHandler.linkUp(link);
+        } else {
+            Device device = deviceService.getDevice(link.src().deviceId());
+            if (device != null) {
+                log.warn("processLinkAdded: Link Added "
+                        + "Notification without Device Added "
+                        + "event, still handling it");
+                processDeviceAdded(device);
+                groupHandler = groupHandlerMap.get(link.src()
+                                                   .deviceId());
                 groupHandler.linkUp(link);
             }
         }
-        defaultRoutingHandler.populateRoutingRulesForLinkStatusChange(null);
+
+        //defaultRoutingHandler.populateRoutingRulesForLinkStatusChange(null);
+        log.trace("processLinkAdded: re-starting route population process");
+        defaultRoutingHandler.startPopulationProcess();
     }
 
     private void processLinkRemoved(Link link) {
@@ -308,20 +373,27 @@
         if (groupHandler != null) {
             groupHandler.portDown(link.src().port());
         }
-        defaultRoutingHandler.populateRoutingRulesForLinkStatusChange(link);
-    }
-
-    private void processGroupAdded(Group group) {
-        log.debug("A new group with ID {} was added", group.id());
-        defaultRoutingHandler.resumePopulationProcess();
+        //defaultRoutingHandler.populateRoutingRulesForLinkStatusChange(link);
+        log.trace("processLinkRemoved: re-starting route population process");
+        defaultRoutingHandler.startPopulationProcess();
     }
 
     private void processDeviceAdded(Device device) {
         log.debug("A new device with ID {} was added", device.id());
-        defaultRoutingHandler.populateTtpRules(device.id());
-        DefaultGroupHandler dgh = DefaultGroupHandler.createGroupHandler(device
-                .id(), appId, deviceConfiguration, linkService, flowObjectiveService);
+        //Irrespective whether the local is a MASTER or not for this device,
+        //create group handler instance and push default TTP flow rules.
+        //Because in a multi-instance setup, instances can initiate
+        //groups for any devices. Also the default TTP rules are needed
+        //to be pushed before inserting any IP table entries for any device
+        DefaultGroupHandler dgh = DefaultGroupHandler.
+                createGroupHandler(device.id(),
+                                   appId,
+                                   deviceConfiguration,
+                                   linkService,
+                                   flowObjectiveService,
+                                   nsNextObjStore);
         groupHandlerMap.put(device.id(), dgh);
+        defaultRoutingHandler.populateTtpRules(device.id());
     }
 
     private void processPortRemoved(Device device, Port port) {
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java
index 4159d3c..41cf8ac 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java
@@ -19,16 +19,12 @@
 import java.util.List;
 import java.util.Set;
 
-import org.onlab.packet.MplsLabel;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Link;
-import org.onosproject.net.flow.DefaultTrafficTreatment;
-import org.onosproject.net.flow.TrafficTreatment;
-import org.onosproject.net.flowobjective.DefaultNextObjective;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
-import org.onosproject.net.flowobjective.NextObjective;
 import org.onosproject.net.link.LinkService;
+import org.onosproject.store.service.EventuallyConsistentMap;
 
 /**
  * Default ECMP group handler creation module for an edge device.
@@ -53,8 +49,11 @@
                                   ApplicationId appId,
                                   DeviceProperties config,
                                   LinkService linkService,
-                                  FlowObjectiveService flowObjService) {
-        super(deviceId, appId, config, linkService, flowObjService);
+                                  FlowObjectiveService flowObjService,
+                                  EventuallyConsistentMap<
+                                  NeighborSetNextObjectiveStoreKey,
+                                  Integer> nsNextObjStore) {
+        super(deviceId, appId, config, linkService, flowObjService, nsNextObjStore);
     }
 
     @Override
@@ -108,7 +107,7 @@
 
     @Override
     protected void newPortToExistingNeighbor(Link newNeighborLink) {
-        log.debug("New port to existing neighbor: Updating "
+        /*log.debug("New port to existing neighbor: Updating "
                 + "groups for edge device {}", deviceId);
         addNeighborAtPort(newNeighborLink.dst().deviceId(),
                           newNeighborLink.src().port());
@@ -129,7 +128,7 @@
                                  mplsLabel(ns.getEdgeLabel()));
             }
 
-            Integer nextId = deviceNextObjectiveIds.get(getGroupKey(ns));
+            Integer nextId = deviceNextObjectiveIds.get(ns);
             if (nextId != null) {
                 NextObjective.Builder nextObjBuilder = DefaultNextObjective
                         .builder().withId(nextId)
@@ -140,7 +139,7 @@
                 NextObjective nextObjective = nextObjBuilder.add();
                 flowObjectiveService.next(deviceId, nextObjective);
             }
-        }
+        }*/
     }
 
     @Override
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
index 1d9f4ec..d9e6b2c 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
@@ -26,6 +26,7 @@
 import java.util.List;
 import java.util.Random;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.MplsLabel;
@@ -42,6 +43,7 @@
 import org.onosproject.net.group.DefaultGroupKey;
 import org.onosproject.net.group.GroupKey;
 import org.onosproject.net.link.LinkService;
+import org.onosproject.store.service.EventuallyConsistentMap;
 import org.slf4j.Logger;
 
 /**
@@ -66,8 +68,10 @@
             new HashMap<DeviceId, Set<PortNumber>>();
     protected HashMap<PortNumber, DeviceId> portDeviceMap =
             new HashMap<PortNumber, DeviceId>();
-    protected HashMap<GroupKey, Integer> deviceNextObjectiveIds =
-            new HashMap<GroupKey, Integer>();
+    //protected HashMap<NeighborSet, Integer> deviceNextObjectiveIds =
+    //        new HashMap<NeighborSet, Integer>();
+    protected EventuallyConsistentMap<
+        NeighborSetNextObjectiveStoreKey, Integer> nsNextObjStore = null;
     protected Random rand = new Random();
 
     protected KryoNamespace.Builder kryo = new KryoNamespace.Builder()
@@ -81,7 +85,10 @@
     protected DefaultGroupHandler(DeviceId deviceId, ApplicationId appId,
                                   DeviceProperties config,
                                   LinkService linkService,
-                                  FlowObjectiveService flowObjService) {
+                                  FlowObjectiveService flowObjService,
+                                  EventuallyConsistentMap<
+                                  NeighborSetNextObjectiveStoreKey,
+                                  Integer> nsNextObjStore) {
         this.deviceId = checkNotNull(deviceId);
         this.appId = checkNotNull(appId);
         this.deviceConfig = checkNotNull(config);
@@ -91,6 +98,7 @@
         isEdgeRouter = config.isEdgeDevice(deviceId);
         nodeMacAddr = checkNotNull(config.getDeviceMac(deviceId));
         this.flowObjectiveService = flowObjService;
+        this.nsNextObjStore = nsNextObjStore;
 
         populateNeighborMaps();
     }
@@ -111,13 +119,20 @@
                                                          ApplicationId appId,
                                                          DeviceProperties config,
                                                          LinkService linkService,
-                                                         FlowObjectiveService flowObjService) {
+                                                         FlowObjectiveService flowObjService,
+                                                         EventuallyConsistentMap<
+                                                         NeighborSetNextObjectiveStoreKey,
+                                                         Integer> nsNextObjStore) {
         if (config.isEdgeDevice(deviceId)) {
             return new DefaultEdgeGroupHandler(deviceId, appId, config,
-                                               linkService, flowObjService);
+                                               linkService,
+                                               flowObjService,
+                                               nsNextObjStore);
         } else {
             return new DefaultTransitGroupHandler(deviceId, appId, config,
-                                                  linkService, flowObjService);
+                                                  linkService,
+                                                  flowObjService,
+                                                  nsNextObjStore);
         }
     }
 
@@ -150,12 +165,56 @@
 
         log.debug("Device {} linkUp at local port {} to neighbor {}", deviceId,
                   newLink.src().port(), newLink.dst().deviceId());
-        if (devicePortMap.get(newLink.dst().deviceId()) == null) {
+        addNeighborAtPort(newLink.dst().deviceId(),
+                          newLink.src().port());
+        /*if (devicePortMap.get(newLink.dst().deviceId()) == null) {
             // New Neighbor
             newNeighbor(newLink);
         } else {
             // Old Neighbor
             newPortToExistingNeighbor(newLink);
+        }*/
+        Set<NeighborSet> nsSet = nsNextObjStore.keySet()
+                .stream()
+                .filter((nsStoreEntry) -> (nsStoreEntry.deviceId().equals(deviceId)))
+                .map((nsStoreEntry) -> (nsStoreEntry.neighborSet()))
+                .filter((ns) -> (ns.getDeviceIds()
+                        .contains(newLink.dst().deviceId())))
+                .collect(Collectors.toSet());
+        log.trace("linkUp: nsNextObjStore contents for device {}:",
+                  deviceId,
+                  nsSet);
+        for (NeighborSet ns : nsSet) {
+            // Create the new bucket to be updated
+            TrafficTreatment.Builder tBuilder =
+                    DefaultTrafficTreatment.builder();
+            tBuilder.setOutput(newLink.src().port())
+                    .setEthDst(deviceConfig.getDeviceMac(
+                          newLink.dst().deviceId()))
+                    .setEthSrc(nodeMacAddr);
+            if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
+                tBuilder.pushMpls()
+                        .setMpls(MplsLabel.
+                                 mplsLabel(ns.getEdgeLabel()));
+            }
+
+            Integer nextId = nsNextObjStore.
+                    get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
+            if (nextId != null) {
+                NextObjective.Builder nextObjBuilder = DefaultNextObjective
+                        .builder().withId(nextId)
+                        .withType(NextObjective.Type.HASHED).fromApp(appId);
+
+                nextObjBuilder.addTreatment(tBuilder.build());
+
+                log.debug("linkUp in device {}: Adding Bucket "
+                        + "with Port {} to next object id {}",
+                        deviceId,
+                        newLink.src().port(),
+                        nextId);
+                NextObjective nextObjective = nextObjBuilder.add();
+                flowObjectiveService.next(deviceId, nextObjective);
+            }
         }
     }
 
@@ -171,10 +230,20 @@
         }
         log.debug("Device {} portDown {} to neighbor {}", deviceId, port,
                   portDeviceMap.get(port));
-        Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(portDeviceMap
+        /*Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(portDeviceMap
                                                                                 .get(port),
                                                                         devicePortMap
-                                                                                .keySet());
+                                                                                .keySet());*/
+        Set<NeighborSet> nsSet = nsNextObjStore.keySet()
+                .stream()
+                .filter((nsStoreEntry) -> (nsStoreEntry.deviceId().equals(deviceId)))
+                .map((nsStoreEntry) -> (nsStoreEntry.neighborSet()))
+                .filter((ns) -> (ns.getDeviceIds()
+                        .contains(portDeviceMap.get(port))))
+                .collect(Collectors.toSet());
+        log.trace("portDown: nsNextObjStore contents for device {}:",
+                  deviceId,
+                  nsSet);
         for (NeighborSet ns : nsSet) {
             // Create the bucket to be removed
             TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
@@ -187,13 +256,19 @@
                                                     .getEdgeLabel()));
             }
 
-            Integer nextId = deviceNextObjectiveIds.get(getGroupKey(ns));
+            Integer nextId = nsNextObjStore.
+                    get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
             if (nextId != null) {
                 NextObjective.Builder nextObjBuilder = DefaultNextObjective
                         .builder().withType(NextObjective.Type.SIMPLE).withId(nextId).fromApp(appId);
 
                 nextObjBuilder.addTreatment(tBuilder.build());
 
+                log.debug("portDown in device {}: Removing Bucket "
+                        + "with Port {} to next object id {}",
+                        deviceId,
+                        port,
+                        nextId);
                 NextObjective nextObjective = nextObjBuilder.remove();
 
                 flowObjectiveService.next(deviceId, nextObjective);
@@ -214,14 +289,31 @@
      * @return int if found or -1
      */
     public int getNextObjectiveId(NeighborSet ns) {
-        Integer nextId = deviceNextObjectiveIds.get(getGroupKey(ns));
+        Integer nextId = nsNextObjStore.
+                get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
         if (nextId == null) {
+            log.trace("getNextObjectiveId in device{}: Next objective id "
+                    + "not found for {} and creating", deviceId, ns);
+            log.trace("getNextObjectiveId: nsNextObjStore contents for device {}: {}",
+                      deviceId,
+                      nsNextObjStore.entrySet()
+                      .stream()
+                      .filter((nsStoreEntry) ->
+                      (nsStoreEntry.getKey().deviceId().equals(deviceId)))
+                      .collect(Collectors.toList()));
             createGroupsFromNeighborsets(Collections.singleton(ns));
-            nextId = deviceNextObjectiveIds.get(getGroupKey(ns));
+            nextId = nsNextObjStore.
+                    get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
             if (nextId == null) {
                 log.warn("getNextObjectiveId: unable to create next objective");
                 return -1;
+            } else {
+                log.debug("getNextObjectiveId in device{}: Next objective id {} "
+                    + "created for {}", deviceId, nextId.intValue(), ns);
             }
+        } else {
+            log.trace("getNextObjectiveId in device{}: Next objective id {} "
+                    + "found for {}", deviceId, nextId.intValue(), ns);
         }
         return nextId.intValue();
     }
@@ -338,6 +430,10 @@
                 if (devicePortMap.get(d) == null) {
                     log.warn("Device {} is not in the port map yet", d);
                     return;
+                } else if (devicePortMap.get(d).size() == 0) {
+                    log.warn("There are no ports for "
+                            + "the Device {} in the port map yet", d);
+                    return;
                 }
 
                 for (PortNumber sp : devicePortMap.get(d)) {
@@ -356,7 +452,11 @@
 
             NextObjective nextObj = nextObjBuilder.add();
             flowObjectiveService.next(deviceId, nextObj);
-            deviceNextObjectiveIds.put(getGroupKey(ns), nextId);
+            log.debug("createGroupsFromNeighborsets: Submited "
+                    + "next objective {} in device {}",
+                    nextId, deviceId);
+            nsNextObjStore.put(new NeighborSetNextObjectiveStoreKey(deviceId, ns),
+                               nextId);
         }
     }
 
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java
index 752ee2d..e12426c 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java
@@ -18,16 +18,12 @@
 import java.util.HashSet;
 import java.util.Set;
 
-import org.onlab.packet.MplsLabel;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Link;
-import org.onosproject.net.flow.DefaultTrafficTreatment;
-import org.onosproject.net.flow.TrafficTreatment;
-import org.onosproject.net.flowobjective.DefaultNextObjective;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
-import org.onosproject.net.flowobjective.NextObjective;
 import org.onosproject.net.link.LinkService;
+import org.onosproject.store.service.EventuallyConsistentMap;
 
 /**
  * Default ECMP group handler creation module for a transit device.
@@ -47,8 +43,11 @@
                                   ApplicationId appId,
                                   DeviceProperties config,
                                   LinkService linkService,
-                                  FlowObjectiveService flowObjService) {
-        super(deviceId, appId, config, linkService, flowObjService);
+                                  FlowObjectiveService flowObjService,
+                                  EventuallyConsistentMap<
+                                  NeighborSetNextObjectiveStoreKey,
+                                  Integer> nsNextObjStore) {
+        super(deviceId, appId, config, linkService, flowObjService, nsNextObjStore);
     }
 
     @Override
@@ -96,7 +95,7 @@
 
     @Override
     protected void newPortToExistingNeighbor(Link newNeighborLink) {
-        log.debug("New port to existing neighbor: Updating "
+        /*log.debug("New port to existing neighbor: Updating "
                 + "groups for transit device {}", deviceId);
         addNeighborAtPort(newNeighborLink.dst().deviceId(),
                           newNeighborLink.src().port());
@@ -118,7 +117,7 @@
             }
 
 
-            Integer nextId = deviceNextObjectiveIds.get(getGroupKey(ns));
+            Integer nextId = deviceNextObjectiveIds.get(ns);
             if (nextId != null) {
                 NextObjective.Builder nextObjBuilder = DefaultNextObjective
                         .builder().withId(nextId)
@@ -129,7 +128,7 @@
                 NextObjective nextObjective = nextObjBuilder.add();
                 flowObjectiveService.next(deviceId, nextObjective);
             }
-        }
+        }*/
     }
 
     @Override
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/NeighborSetNextObjectiveStoreKey.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/NeighborSetNextObjectiveStoreKey.java
new file mode 100644
index 0000000..9ace531
--- /dev/null
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/NeighborSetNextObjectiveStoreKey.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.segmentrouting.grouphandler;
+
+import java.util.Objects;
+
+import org.onosproject.net.DeviceId;
+
+/**
+ * Class definition of Key for Neighborset to NextObjective store.
+ */
+public class NeighborSetNextObjectiveStoreKey {
+    private final DeviceId deviceId;
+    private final NeighborSet ns;
+
+    public NeighborSetNextObjectiveStoreKey(DeviceId deviceId,
+                                            NeighborSet ns) {
+        this.deviceId = deviceId;
+        this.ns = ns;
+    }
+
+    public DeviceId deviceId() {
+        return this.deviceId;
+    }
+
+    public NeighborSet neighborSet() {
+        return this.ns;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof NeighborSetNextObjectiveStoreKey)) {
+            return false;
+        }
+        NeighborSetNextObjectiveStoreKey that =
+                (NeighborSetNextObjectiveStoreKey) o;
+        return (Objects.equals(this.deviceId, that.deviceId) &&
+                Objects.equals(this.ns, that.ns));
+    }
+
+    // The list of neighbor ids and label are used for comparison.
+    @Override
+    public int hashCode() {
+        int result = 17;
+        result = 31 * result + Objects.hashCode(this.deviceId)
+                + Objects.hashCode(this.ns);
+
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "Device: " + deviceId + " Neighborset: " + ns;
+    }
+}
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
index a2a1b09..6623696 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
@@ -27,6 +27,7 @@
 import org.onlab.packet.MplsLabel;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.segmentrouting.grouphandler.GroupBucketIdentifier.BucketOutputType;
+import org.onosproject.store.service.EventuallyConsistentMap;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
@@ -58,8 +59,11 @@
                               ApplicationId appId,
                               DeviceProperties config,
                               LinkService linkService,
-                              FlowObjectiveService flowObjService) {
-        super(deviceId, appId, config, linkService, flowObjService);
+                              FlowObjectiveService flowObjService,
+                              EventuallyConsistentMap<
+                              NeighborSetNextObjectiveStoreKey,
+                              Integer> nsNextObjStore) {
+        super(deviceId, appId, config, linkService, flowObjService, nsNextObjStore);
     }
 
     public PolicyGroupIdentifier createPolicyGroupChain(String id,