CORD-48 Implementation of hashing Next Objective in OF-DPA driver. Major changes to ensure multi-ONOS-instance group-chain installation.
Also includes:
     Changes to Next Objective that adds metadata field for applications to optionally send auxillary info to drivers
     Changes to Next Objective that allows more explicit modification of the next objective
     Changes to Forwarding Objective and PendingNext to include hashCode() and equals() method
     MplsBosInstruction included in kryo serializer
     GroupKey's byte[] represented as a hex string
     Bug fix in mpls flow installation to report failure in install
     Bug fix in linkUp in SR app to disallow non-masters to modify groups
     Bug fix in ordering of actions in group

Change-Id: I3e7003f55724c2de79589e43e11d05ff4815a81d
diff --git a/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java b/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
index a737339..9922587 100644
--- a/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
@@ -448,7 +448,6 @@
         if (nextHops.isEmpty()) {
             nextHops.add(destSw);
         }
-
         // If both target switch and dest switch are edge routers, then set IP
         // rule for both subnet and router IP.
         boolean targetIsEdge;
@@ -467,7 +466,7 @@
         if (targetIsEdge && destIsEdge) {
             Set<Ip4Prefix> subnets = config.getSubnets(destSw);
             log.debug("* populateEcmpRoutingRulePartial in device {} towards {} for subnets {}",
-                    targetSw, destSw, subnets);
+                      targetSw, destSw, subnets);
             result = rulePopulator.populateIpRuleForSubnet(targetSw,
                                                            subnets,
                                                            destSw,
@@ -479,24 +478,23 @@
             Ip4Address routerIp = destRouterIp;
             IpPrefix routerIpPrefix = IpPrefix.valueOf(routerIp, IpPrefix.MAX_INET_MASK_LENGTH);
             log.debug("* populateEcmpRoutingRulePartial in device {} towards {} for router IP {}",
-                    targetSw, destSw, routerIpPrefix);
+                      targetSw, destSw, routerIpPrefix);
             result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix, destSw, nextHops);
             if (!result) {
                 return false;
             }
 
-        // If the target switch is an edge router, then set IP rules for the router IP.
         } else if (targetIsEdge) {
+            // If the target switch is an edge router, then set IP rules for the router IP.
             Ip4Address routerIp = destRouterIp;
             IpPrefix routerIpPrefix = IpPrefix.valueOf(routerIp, IpPrefix.MAX_INET_MASK_LENGTH);
             log.debug("* populateEcmpRoutingRulePartial in device {} towards {} for router IP {}",
-                    targetSw, destSw, routerIpPrefix);
+                      targetSw, destSw, routerIpPrefix);
             result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix, destSw, nextHops);
             if (!result) {
                 return false;
             }
         }
-
         // Populates MPLS rules to all routers
         log.debug("* populateEcmpRoutingRulePartial in device{} towards {} for all MPLS rules",
                 targetSw, destSw);
@@ -504,7 +502,6 @@
         if (!result) {
             return false;
         }
-
         return true;
     }
 
diff --git a/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java b/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
index bc3ce8c..a07a15d 100644
--- a/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
+++ b/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
@@ -15,6 +15,7 @@
  */
 package org.onosproject.segmentrouting;
 
+import org.onlab.packet.EthType;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Prefix;
@@ -26,7 +27,6 @@
 import org.onosproject.segmentrouting.config.DeviceConfiguration;
 import org.onosproject.segmentrouting.grouphandler.NeighborSet;
 import org.onosproject.net.DeviceId;
-import org.onosproject.net.Link;
 import org.onosproject.net.Port;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.flow.DefaultTrafficSelector;
@@ -227,7 +227,15 @@
             treatment = null;
         }
 
-        if (srManager.getNextObjectiveId(deviceId, ns) <= 0) {
+        // setup metadata to pass to nextObjective - indicate the vlan on egress
+        // if needed by the switch pipeline. Since neighbor sets are always to
+        // other neighboring routers, there is no subnet assigned on those ports.
+        TrafficSelector.Builder metabuilder = DefaultTrafficSelector.builder(selector);
+        metabuilder.matchVlanId(
+            VlanId.vlanId(SegmentRoutingManager.ASSIGNED_VLAN_NO_SUBNET));
+
+        int nextId = srManager.getNextObjectiveId(deviceId, ns, metabuilder.build());
+        if (nextId <= 0) {
             log.warn("No next objective in {} for ns: {}", deviceId, ns);
             return false;
         }
@@ -236,7 +244,7 @@
                 .builder()
                 .fromApp(srManager.appId)
                 .makePermanent()
-                .nextStep(srManager.getNextObjectiveId(deviceId, ns))
+                .nextStep(nextId)
                 .withSelector(selector)
                 .withPriority(100)
                 .withFlag(ForwardingObjective.Flag.SPECIFIC);
@@ -279,63 +287,70 @@
         List<ForwardingObjective.Builder> fwdObjBuilders = new ArrayList<>();
 
         // TODO Handle the case of Bos == false
-        sbuilder.matchMplsLabel(MplsLabel.mplsLabel(segmentId));
         sbuilder.matchEthType(Ethernet.MPLS_UNICAST);
+        sbuilder.matchMplsLabel(MplsLabel.mplsLabel(segmentId));
+        TrafficSelector selector = sbuilder.build();
 
-        // If the next hop is the destination router, do PHP
+        // setup metadata to pass to nextObjective - indicate the vlan on egress
+        // if needed by the switch pipeline. Since mpls next-hops are always to
+        // other neighboring routers, there is no subnet assigned on those ports.
+        TrafficSelector.Builder metabuilder = DefaultTrafficSelector.builder(selector);
+        metabuilder.matchVlanId(
+            VlanId.vlanId(SegmentRoutingManager.ASSIGNED_VLAN_NO_SUBNET));
+
+        // If the next hop is the destination router for the segment, do pop
         if (nextHops.size() == 1 && destSwId.equals(nextHops.toArray()[0])) {
             log.debug("populateMplsRule: Installing MPLS forwarding objective for "
-                    + "label {} in switch {} with PHP",
-                    segmentId,
-                    deviceId);
+                    + "label {} in switch {} with pop", segmentId, deviceId);
 
+            // bos pop case (php)
             ForwardingObjective.Builder fwdObjBosBuilder =
                     getMplsForwardingObjective(deviceId,
-                                               destSwId,
                                                nextHops,
                                                true,
-                                               true);
-            // TODO: Check with Sangho on why we need this
-            ForwardingObjective.Builder fwdObjNoBosBuilder =
-                    getMplsForwardingObjective(deviceId,
-                                               destSwId,
-                                               nextHops,
                                                true,
-                                               false);
-            if (fwdObjBosBuilder != null) {
-                fwdObjBuilders.add(fwdObjBosBuilder);
-            } else {
-                log.warn("Failed to set MPLS rules.");
+                                               metabuilder.build());
+            if (fwdObjBosBuilder == null) {
                 return false;
             }
+            fwdObjBuilders.add(fwdObjBosBuilder);
+
+            // XXX not-bos pop case,  SR app multi-label not implemented yet
+            /*ForwardingObjective.Builder fwdObjNoBosBuilder =
+                    getMplsForwardingObjective(deviceId,
+                                               nextHops,
+                                               true,
+                                               false);*/
+
         } else {
+            // next hop is not destination, SR CONTINUE case (swap with self)
             log.debug("Installing MPLS forwarding objective for "
-                    + "label {} in switch {} without PHP",
-                    segmentId,
-                    deviceId);
+                    + "label {} in switch {} without pop", segmentId, deviceId);
 
+            // continue case with bos - this does get triggered in edge routers
+            // and in core routers - driver can handle depending on availability
+            // of MPLS ECMP or not
             ForwardingObjective.Builder fwdObjBosBuilder =
                     getMplsForwardingObjective(deviceId,
-                                               destSwId,
                                                nextHops,
                                                false,
-                                               true);
-            // TODO: Check with Sangho on why we need this
-            ForwardingObjective.Builder fwdObjNoBosBuilder =
-                    getMplsForwardingObjective(deviceId,
-                                               destSwId,
-                                               nextHops,
-                                               false,
-                                               false);
-            if (fwdObjBosBuilder != null) {
-                fwdObjBuilders.add(fwdObjBosBuilder);
-            } else {
-                log.warn("Failed to set MPLS rules.");
+                                               true,
+                                               metabuilder.build());
+            if (fwdObjBosBuilder == null) {
                 return false;
             }
+            fwdObjBuilders.add(fwdObjBosBuilder);
+
+            // XXX continue case with not-bos - SR app multi label not implemented yet
+            // also requires MPLS ECMP
+            /*ForwardingObjective.Builder fwdObjNoBosBuilder =
+                    getMplsForwardingObjective(deviceId,
+                                               nextHops,
+                                               false,
+                                               false); */
+
         }
 
-        TrafficSelector selector = sbuilder.build();
         for (ForwardingObjective.Builder fwdObjBuilder : fwdObjBuilders) {
             ((Builder) ((Builder) fwdObjBuilder.fromApp(srManager.appId)
                     .makePermanent()).withSelector(selector)
@@ -345,76 +360,61 @@
                 forward(deviceId,
                         fwdObjBuilder.
                         add(new SRObjectiveContext(deviceId,
-                                                   SRObjectiveContext.ObjectiveType.FORWARDING)));
+                                    SRObjectiveContext.ObjectiveType.FORWARDING)));
             rulePopulationCounter.incrementAndGet();
         }
 
         return true;
     }
 
-    private ForwardingObjective.Builder getMplsForwardingObjective(DeviceId deviceId,
-                                                                   DeviceId destSw,
-                                                                   Set<DeviceId> nextHops,
-                                                                   boolean phpRequired,
-                                                                   boolean isBos) {
+    private ForwardingObjective.Builder getMplsForwardingObjective(
+                                             DeviceId deviceId,
+                                             Set<DeviceId> nextHops,
+                                             boolean phpRequired,
+                                             boolean isBos,
+                                             TrafficSelector meta) {
+
         ForwardingObjective.Builder fwdBuilder = DefaultForwardingObjective
                 .builder().withFlag(ForwardingObjective.Flag.SPECIFIC);
-        DeviceId nextHop = (DeviceId) nextHops.toArray()[0];
-
-        boolean isEdge;
-        MacAddress srcMac;
-        MacAddress dstMac;
-        try {
-            isEdge = config.isEdgeDevice(deviceId);
-            srcMac = config.getDeviceMac(deviceId);
-            dstMac = config.getDeviceMac(nextHop);
-        } catch (DeviceConfigNotFoundException e) {
-            log.warn(e.getMessage() + " Aborting getMplsForwardingObjective");
-            return null;
-        }
 
         TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
 
         if (phpRequired) {
+            // php case - pop should always be flow-action
             log.debug("getMplsForwardingObjective: php required");
             tbuilder.deferred().copyTtlIn();
             if (isBos) {
-                tbuilder.deferred().popMpls(Ethernet.TYPE_IPV4).decNwTtl();
+                tbuilder.deferred().popMpls(EthType.EtherType.IPV4.ethType())
+                    .decNwTtl();
             } else {
-                tbuilder.deferred().popMpls(Ethernet.MPLS_UNICAST).decMplsTtl();
+                tbuilder.deferred().popMpls(EthType.EtherType.MPLS_UNICAST.ethType())
+                    .decMplsTtl();
             }
         } else {
+            // swap with self case - SR CONTINUE
             log.debug("getMplsForwardingObjective: php not required");
             tbuilder.deferred().decMplsTtl();
         }
 
-        if (!isECMPSupportedInTransitRouter() && !isEdge) {
-            PortNumber port = selectOnePort(deviceId, nextHops);
-            if (port == null) {
-                log.warn("No link from {} to {}", deviceId, nextHops);
-                return null;
-            }
-            tbuilder.deferred()
-                    .setEthSrc(srcMac)
-                    .setEthDst(dstMac)
-                    .setOutput(port);
-            fwdBuilder.withTreatment(tbuilder.build());
-        } else {
-            NeighborSet ns = new NeighborSet(nextHops);
-            fwdBuilder.withTreatment(tbuilder.build());
-            fwdBuilder.nextStep(srManager
-                    .getNextObjectiveId(deviceId, ns));
+        // All forwarding is via ECMP group, the metadata informs the driver
+        // that the next-Objective will be used by MPLS flows. In other words,
+        // MPLS ECMP is requested. It is up to the driver to decide if these
+        // packets will be hashed or not.
+        fwdBuilder.withTreatment(tbuilder.build());
+        NeighborSet ns = new NeighborSet(nextHops);
+        log.debug("Trying to get a nextObjid for mpls rule on device:{} to ns:{}",
+                 deviceId, ns);
+
+        int nextId = srManager.getNextObjectiveId(deviceId, ns, meta);
+        if (nextId <= 0) {
+            log.warn("No next objective in {} for ns: {}", deviceId, ns);
+            return null;
         }
 
+        fwdBuilder.nextStep(nextId);
         return fwdBuilder;
     }
 
-    private boolean isECMPSupportedInTransitRouter() {
-
-        // TODO: remove this function when objectives subsystem is supported.
-        return false;
-    }
-
     /**
      * Creates a filtering objective to permit all untagged packets with a
      * dstMac corresponding to the router's MAC address. For those pipelines
@@ -552,22 +552,6 @@
     }
 
 
-    private PortNumber selectOnePort(DeviceId srcId, Set<DeviceId> destIds) {
-
-        Set<Link> links = srManager.linkService.getDeviceLinks(srcId);
-        for (DeviceId destId: destIds) {
-            for (Link link : links) {
-                if (link.dst().deviceId().equals(destId)) {
-                    return link.src().port();
-                } else if (link.src().deviceId().equals(destId)) {
-                    return link.dst().port();
-                }
-            }
-        }
-
-        return null;
-    }
-
     private static class SRObjectiveContext implements ObjectiveContext {
         enum ObjectiveType {
             FILTER,
diff --git a/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java b/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
index 84fe516..f6bf649 100644
--- a/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
+++ b/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
@@ -424,17 +424,22 @@
 
     /**
      * Returns the next objective ID for the given NeighborSet.
-     * If the nextObjectiveID does not exist, a new one is created and returned.
+     * If the nextObjective does not exist, a new one is created and
+     * it's id is returned.
+     * TODO move the side-effect creation of a Next Objective into a new method
      *
      * @param deviceId Device ID
      * @param ns NegighborSet
-     * @return next objective ID
+     * @param meta metadata passed into the creation of a Next Objective
+     * @return next objective ID or -1 if an error was encountered during the
+     *         creation of the nextObjective
      */
-    public int getNextObjectiveId(DeviceId deviceId, NeighborSet ns) {
+    public int getNextObjectiveId(DeviceId deviceId, NeighborSet ns,
+                                  TrafficSelector meta) {
         if (groupHandlerMap.get(deviceId) != null) {
             log.trace("getNextObjectiveId query in device {}", deviceId);
             return groupHandlerMap
-                    .get(deviceId).getNextObjectiveId(ns);
+                    .get(deviceId).getNextObjectiveId(ns, meta);
         } else {
             log.warn("getNextObjectiveId query in device {} not found", deviceId);
             return -1;
@@ -586,7 +591,8 @@
         DefaultGroupHandler groupHandler = groupHandlerMap.get(link.src()
                 .deviceId());
         if (groupHandler != null) {
-            groupHandler.linkUp(link);
+            groupHandler.linkUp(link, mastershipService.isLocalMaster(
+                                           link.src().deviceId()));
         } else {
             Device device = deviceService.getDevice(link.src().deviceId());
             if (device != null) {
@@ -596,7 +602,7 @@
                 processDeviceAdded(device);
                 groupHandler = groupHandlerMap.get(link.src()
                                                    .deviceId());
-                groupHandler.linkUp(link);
+                groupHandler.linkUp(link, mastershipService.isLocalMaster(device.id()));
             }
         }
 
diff --git a/src/main/java/org/onosproject/segmentrouting/TunnelHandler.java b/src/main/java/org/onosproject/segmentrouting/TunnelHandler.java
index 7d025c7..b86adad 100644
--- a/src/main/java/org/onosproject/segmentrouting/TunnelHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/TunnelHandler.java
@@ -194,7 +194,7 @@
             tunnel.allowToRemoveGroup(true);
         }
 
-        return groupHandlerMap.get(deviceId).getNextObjectiveId(ns);
+        return groupHandlerMap.get(deviceId).getNextObjectiveId(ns, null);
     }
 
 }
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java
index 33496bd..6b6d960 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java
@@ -93,7 +93,7 @@
                 + "with label for sw {} is {}",
                 deviceId, nsSet);
 
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
 
     @Override
@@ -107,7 +107,7 @@
         Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(
                                              newNeighborLink.dst().deviceId(),
                                              devicePortMap.keySet());
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
 
     @Override
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
index b394db5..e792bf6 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
@@ -21,11 +21,11 @@
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
 import org.onlab.packet.Ip4Prefix;
@@ -38,6 +38,7 @@
 import org.onosproject.net.Link;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flowobjective.DefaultNextObjective;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
@@ -71,10 +72,10 @@
     protected LinkService linkService;
     protected FlowObjectiveService flowObjectiveService;
 
-    protected HashMap<DeviceId, Set<PortNumber>> devicePortMap = new HashMap<>();
-    protected HashMap<PortNumber, DeviceId> portDeviceMap = new HashMap<>();
-    //protected HashMap<NeighborSet, Integer> deviceNextObjectiveIds =
-    //        new HashMap<NeighborSet, Integer>();
+    protected ConcurrentHashMap<DeviceId, Set<PortNumber>> devicePortMap =
+            new ConcurrentHashMap<>();
+    protected ConcurrentHashMap<PortNumber, DeviceId> portDeviceMap =
+            new ConcurrentHashMap<>();
     protected EventuallyConsistentMap<
         NeighborSetNextObjectiveStoreKey, Integer> nsNextObjStore = null;
     protected EventuallyConsistentMap<
@@ -173,7 +174,7 @@
      *
      * @param newLink new neighbor link
      */
-    public void linkUp(Link newLink) {
+    public void linkUp(Link newLink, boolean isMaster) {
 
         if (newLink.type() != Link.Type.DIRECT) {
             log.warn("linkUp: unknown link type");
@@ -186,6 +187,8 @@
             return;
         }
 
+        log.info("* LinkUP: Device {} linkUp at local port {} to neighbor {}", deviceId,
+                 newLink.src().port(), newLink.dst().deviceId());
         MacAddress dstMac;
         try {
             dstMac = deviceConfig.getDeviceMac(newLink.dst().deviceId());
@@ -194,8 +197,6 @@
             return;
         }
 
-        log.debug("Device {} linkUp at local port {} to neighbor {}", deviceId,
-                  newLink.src().port(), newLink.dst().deviceId());
         addNeighborAtPort(newLink.dst().deviceId(),
                           newLink.src().port());
         /*if (devicePortMap.get(newLink.dst().deviceId()) == null) {
@@ -237,14 +238,20 @@
 
                 nextObjBuilder.addTreatment(tBuilder.build());
 
-                log.debug("linkUp in device {}: Adding Bucket "
-                        + "with Port {} to next object id {}",
+                log.info("**linkUp in device {}: Adding Bucket "
+                        + "with Port {} to next object id {} and amIMaster:{}",
                         deviceId,
                         newLink.src().port(),
-                        nextId);
-                NextObjective nextObjective = nextObjBuilder.
-                        add(new SRNextObjectiveContext(deviceId));
-                flowObjectiveService.next(deviceId, nextObjective);
+                        nextId, isMaster);
+
+                if (isMaster) {
+                    NextObjective nextObjective = nextObjBuilder.
+                            addToExisting(new SRNextObjectiveContext(deviceId));
+                    flowObjectiveService.next(deviceId, nextObjective);
+                }
+            } else {
+                log.warn("linkUp in device {}, but global store has no record "
+                        + "for neighbor-set {}", deviceId, ns);
             }
         }
     }
@@ -305,15 +312,16 @@
 
                 nextObjBuilder.addTreatment(tBuilder.build());
 
-                log.debug("portDown in device {}: Removing Bucket "
+                log.info("**portDown in device {}: Removing Bucket "
                         + "with Port {} to next object id {}",
                         deviceId,
                         port,
                         nextId);
-                NextObjective nextObjective = nextObjBuilder.
+                // should do removefromexisting and only if master
+                /*NextObjective nextObjective = nextObjBuilder.
                         remove(new SRNextObjectiveContext(deviceId));
 
-                flowObjectiveService.next(deviceId, nextObjective);
+                flowObjectiveService.next(deviceId, nextObjective);*/
             }
 
         }
@@ -325,12 +333,15 @@
     /**
      * Returns the next objective associated with the neighborset.
      * If there is no next objective for this neighborset, this API
-     * would create a next objective and return.
+     * would create a next objective and return. Optionally metadata can be
+     * passed in for the creation of the next objective.
      *
      * @param ns neighborset
-     * @return int if found or -1
+     * @param meta metadata passed into the creation of a Next Objective
+     * @return int if found or -1 if there are errors in the creation of the
+     *          neighbor set.
      */
-    public int getNextObjectiveId(NeighborSet ns) {
+    public int getNextObjectiveId(NeighborSet ns, TrafficSelector meta) {
         Integer nextId = nsNextObjStore.
                 get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
         if (nextId == null) {
@@ -343,7 +354,7 @@
                       .filter((nsStoreEntry) ->
                       (nsStoreEntry.getKey().deviceId().equals(deviceId)))
                       .collect(Collectors.toList()));
-            createGroupsFromNeighborsets(Collections.singleton(ns));
+            createGroupsFromNeighborsets(Collections.singleton(ns), meta);
             nextId = nsNextObjStore.
                     get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
             if (nextId == null) {
@@ -421,17 +432,19 @@
         // Update DeviceToPort database
         log.debug("Device {} addNeighborAtPort: neighbor {} at port {}",
                   deviceId, neighborId, portToNeighbor);
-        if (devicePortMap.get(neighborId) != null) {
-            devicePortMap.get(neighborId).add(portToNeighbor);
-        } else {
-            Set<PortNumber> ports = new HashSet<>();
-            ports.add(portToNeighbor);
-            devicePortMap.put(neighborId, ports);
+        Set<PortNumber> ports = Collections
+                .newSetFromMap(new ConcurrentHashMap<PortNumber, Boolean>());
+        ports.add(portToNeighbor);
+        Set<PortNumber> portnums = devicePortMap.putIfAbsent(neighborId, ports);
+        if (portnums != null) {
+            portnums.add(portToNeighbor);
         }
 
         // Update portToDevice database
-        if (portDeviceMap.get(portToNeighbor) == null) {
-            portDeviceMap.put(portToNeighbor, neighborId);
+        DeviceId prev = portDeviceMap.putIfAbsent(portToNeighbor, neighborId);
+        if (prev != null) {
+            log.warn("Device: {} port: {} has neighbor: {}. NOT updating "
+                    + "to neighbor: {}", deviceId, portToNeighbor, prev, neighborId);
         }
     }
 
@@ -505,61 +518,66 @@
      * Creates Groups from a set of NeighborSet given.
      *
      * @param nsSet a set of NeighborSet
+     * @param meta metadata passed into the creation of a Next Objective
      */
-    public void createGroupsFromNeighborsets(Set<NeighborSet> nsSet) {
+    public void createGroupsFromNeighborsets(Set<NeighborSet> nsSet,
+                                             TrafficSelector meta) {
         for (NeighborSet ns : nsSet) {
             int nextId = flowObjectiveService.allocateNextId();
             NextObjective.Builder nextObjBuilder = DefaultNextObjective
                     .builder().withId(nextId)
                     .withType(NextObjective.Type.HASHED).fromApp(appId);
-            for (DeviceId d : ns.getDeviceIds()) {
-                if (devicePortMap.get(d) == null) {
-                    log.warn("Device {} is not in the port map yet", d);
+            for (DeviceId neighborId : ns.getDeviceIds()) {
+                if (devicePortMap.get(neighborId) == null) {
+                    log.warn("Neighbor {} is not in the port map yet for dev:{}",
+                             neighborId, deviceId);
                     return;
-                } else if (devicePortMap.get(d).size() == 0) {
+                } else if (devicePortMap.get(neighborId).size() == 0) {
                     log.warn("There are no ports for "
-                            + "the Device {} in the port map yet", d);
+                            + "the Device {} in the port map yet", neighborId);
                     return;
                 }
 
-                MacAddress deviceMac;
+                MacAddress neighborMac;
                 try {
-                    deviceMac = deviceConfig.getDeviceMac(d);
+                    neighborMac = deviceConfig.getDeviceMac(neighborId);
                 } catch (DeviceConfigNotFoundException e) {
                     log.warn(e.getMessage() + " Aborting createGroupsFromNeighborsets.");
                     return;
                 }
 
-                for (PortNumber sp : devicePortMap.get(d)) {
+                for (PortNumber sp : devicePortMap.get(neighborId)) {
                     TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
                             .builder();
-                    tBuilder.setOutput(sp)
-                            .setEthDst(deviceMac)
+                    tBuilder.setEthDst(neighborMac)
                             .setEthSrc(nodeMacAddr);
                     if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
                         tBuilder.pushMpls()
                                 .copyTtlOut()
                                 .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
                     }
+                    tBuilder.setOutput(sp);
                     nextObjBuilder.addTreatment(tBuilder.build());
                 }
             }
-
+            if (meta != null) {
+                nextObjBuilder.setMeta(meta);
+            }
             NextObjective nextObj = nextObjBuilder.
                     add(new SRNextObjectiveContext(deviceId));
-            flowObjectiveService.next(deviceId, nextObj);
-            log.debug("createGroupsFromNeighborsets: Submited "
-                            + "next objective {} in device {}",
+            log.info("**createGroupsFromNeighborsets: Submited "
+                    + "next objective {} in device {}",
                     nextId, deviceId);
+            flowObjectiveService.next(deviceId, nextObj);
             nsNextObjStore.put(new NeighborSetNextObjectiveStoreKey(deviceId, ns),
                                nextId);
         }
     }
 
+
     public void createGroupsFromSubnetConfig() {
         Map<Ip4Prefix, List<PortNumber>> subnetPortMap =
                 this.deviceConfig.getSubnetPortsMap(this.deviceId);
-
         // Construct a broadcast group for each subnet
         subnetPortMap.forEach((subnet, ports) -> {
             SubnetNextObjectiveStoreKey key =
@@ -612,6 +630,9 @@
                     .withType(NextObjective.Type.HASHED).fromApp(appId);
             NextObjective nextObjective = nextObjBuilder.
                     remove(new SRNextObjectiveContext(deviceId));
+            log.info("**removeGroup: Submited "
+                    + "next objective {} in device {}",
+                    objectiveId, deviceId);
             flowObjectiveService.next(deviceId, nextObjective);
 
             for (Map.Entry<NeighborSetNextObjectiveStoreKey, Integer> entry: nsNextObjStore.entrySet()) {
@@ -634,14 +655,14 @@
         }
         @Override
         public void onSuccess(Objective objective) {
-            log.debug("Next objective operation successful in device {}",
-                      deviceId);
+            log.info("Next objective {} operation successful in device {}",
+                      objective.id(), deviceId);
         }
 
         @Override
         public void onError(Objective objective, ObjectiveError error) {
             log.warn("Next objective {} operation failed with error: {} in device {}",
-                     objective, error, deviceId);
+                     objective.id(), error, deviceId);
         }
     }
 }
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java
index 8e1b6a8..14d77ba 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java
@@ -81,7 +81,7 @@
         log.debug("createGroupsAtTransitRouter: The neighborset with label "
                 + "for sw {} is {}", deviceId, nsSet);
 
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
 
     @Override
@@ -95,7 +95,7 @@
         Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(
                                              newNeighborLink.dst().deviceId(),
                                              devicePortMap.keySet());
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
 
     @Override