CORD-367 L2 bridging and L3 routing support with internal VLANs in OF-DPA.
Also includes:
     All forwarding in app is now via nextObjectives (not treatments) - Spring Open driver converts
     non-ECMP forwarding to flow-actions, while OF-DPA driver continues to use groups.
     Convert 'setMeta' methods to 'withMeta' in Flow Objectives API.
     Bug fix in Flow Objective Manager - set of PendingNext is now threadsafe.
     Bug fix in ArpHandler - now recognizes routerIp in addition to gatewayIps
     Removed a bunch of testcode
     Added group count in CLI

Change-Id: Id3b879c5dda78151ca0ec359179f1604066d39fc
diff --git a/src/main/java/org/onosproject/segmentrouting/ArpHandler.java b/src/main/java/org/onosproject/segmentrouting/ArpHandler.java
index 2c6412c..7f4bcb1 100644
--- a/src/main/java/org/onosproject/segmentrouting/ArpHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/ArpHandler.java
@@ -107,7 +107,7 @@
                                             vlanId);
 
         // ARP request for router. Send ARP reply.
-        if (isArpReqForRouter(deviceId, arpRequest)) {
+        if (isArpForRouter(deviceId, arpRequest)) {
             Ip4Address targetAddress = Ip4Address.valueOf(arpRequest.getTargetProtocolAddress());
             sendArpResponse(arpRequest, config.getRouterMacForAGatewayIp(targetAddress), vlanId);
         } else {
@@ -130,7 +130,7 @@
                                             vlanId);
 
         // ARP reply for router. Process all pending IP packets.
-        if (isArpReqForRouter(deviceId, arpReply)) {
+        if (isArpForRouter(deviceId, arpReply)) {
             Ip4Address hostIpAddress = Ip4Address.valueOf(arpReply.getSenderProtocolAddress());
             srManager.ipHandler.forwardPackets(deviceId, hostIpAddress);
         } else {
@@ -141,7 +141,8 @@
             // ARP reply for unknown host, Flood in the subnet.
             } else {
                 // Don't flood to non-edge ports
-                if (vlanId.equals(VlanId.vlanId(srManager.ASSIGNED_VLAN_NO_SUBNET))) {
+                if (vlanId.equals(
+                        VlanId.vlanId(SegmentRoutingManager.ASSIGNED_VLAN_NO_SUBNET))) {
                     return;
                 }
                 removeVlanAndFlood(payload, inPort);
@@ -150,14 +151,21 @@
     }
 
 
-    private boolean isArpReqForRouter(DeviceId deviceId, ARP arpRequest) {
-        Set<Ip4Address> gatewayIpAddresses = config.getPortIPs(deviceId);
-        if (gatewayIpAddresses != null) {
-            Ip4Address targetProtocolAddress = Ip4Address.valueOf(arpRequest
-                    .getTargetProtocolAddress());
-            if (gatewayIpAddresses.contains(targetProtocolAddress)) {
+    private boolean isArpForRouter(DeviceId deviceId, ARP arpMsg) {
+        Ip4Address targetProtocolAddress = Ip4Address.valueOf(
+                                               arpMsg.getTargetProtocolAddress());
+        Set<Ip4Address> gatewayIpAddresses = null;
+        try {
+            if (targetProtocolAddress.equals(config.getRouterIp(deviceId))) {
                 return true;
             }
+            gatewayIpAddresses = config.getPortIPs(deviceId);
+        } catch (DeviceConfigNotFoundException e) {
+            log.warn(e.getMessage() + " Aborting check for router IP in processing arp");
+        }
+        if (gatewayIpAddresses != null &&
+                gatewayIpAddresses.contains(targetProtocolAddress)) {
+            return true;
         }
         return false;
     }
diff --git a/src/main/java/org/onosproject/segmentrouting/IcmpHandler.java b/src/main/java/org/onosproject/segmentrouting/IcmpHandler.java
index eb3b3fd..d1dc8dd 100644
--- a/src/main/java/org/onosproject/segmentrouting/IcmpHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/IcmpHandler.java
@@ -88,10 +88,10 @@
                 (destinationAddress.equals(routerIpAddress) ||
                         gatewayIpAddresses.contains(destinationAddress))) {
             sendICMPResponse(ethernet, connectPoint);
-            // TODO: do we need to set the flow rule again ??
 
         // ICMP for any known host
         } else if (!srManager.hostService.getHostsByIp(destinationAddress).isEmpty()) {
+            // TODO: known host packet should not be coming to controller - resend flows?
             srManager.ipHandler.forwardPackets(deviceId, destinationAddress);
 
         // ICMP for an unknown host in the subnet of the router
diff --git a/src/main/java/org/onosproject/segmentrouting/IpHandler.java b/src/main/java/org/onosproject/segmentrouting/IpHandler.java
index b1682e7..d6a9dcf 100644
--- a/src/main/java/org/onosproject/segmentrouting/IpHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/IpHandler.java
@@ -98,7 +98,7 @@
      */
     public void addToPacketBuffer(IPv4 ipPacket) {
 
-        // Better not buffer TPC packets due to out-of-order packet transfer
+        // Better not buffer TCP packets due to out-of-order packet transfer
         if (ipPacket.getProtocol() == IPv4.PROTOCOL_TCP) {
             return;
         }
diff --git a/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java b/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
index fc6a3f3..d4aa770 100644
--- a/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
+++ b/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
@@ -147,20 +147,34 @@
         TrafficSelector.Builder sbuilder = DefaultTrafficSelector.builder();
         TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
 
-        sbuilder.matchIPDst(IpPrefix.valueOf(hostIp, IpPrefix.MAX_INET_MASK_LENGTH));
         sbuilder.matchEthType(Ethernet.TYPE_IPV4);
+        sbuilder.matchIPDst(IpPrefix.valueOf(hostIp, IpPrefix.MAX_INET_MASK_LENGTH));
+        TrafficSelector selector = sbuilder.build();
 
         tbuilder.deferred()
                 .setEthDst(hostMac)
                 .setEthSrc(deviceMac)
                 .setOutput(outPort);
-
         TrafficTreatment treatment = tbuilder.build();
-        TrafficSelector selector = sbuilder.build();
+
+        // All forwarding is via Groups. Drivers can re-purpose to flow-actions if needed.
+        // for switch pipelines that need it, provide outgoing vlan as metadata
+        VlanId outvlan = null;
+        Ip4Prefix subnet = srManager.deviceConfiguration.getPortSubnet(deviceId, outPort);
+        if (subnet == null) {
+            outvlan = VlanId.vlanId(SegmentRoutingManager.ASSIGNED_VLAN_NO_SUBNET);
+        } else {
+            outvlan = srManager.getSubnetAssignedVlanId(deviceId, subnet);
+        }
+        TrafficSelector meta = DefaultTrafficSelector.builder()
+                                    .matchVlanId(outvlan).build();
+        int portNextObjId = srManager.getPortNextObjectiveId(deviceId, outPort,
+                                                             treatment, meta);
 
         return DefaultForwardingObjective.builder()
+                .withSelector(selector)
+                .nextStep(portNextObjId)
                 .fromApp(srManager.appId).makePermanent()
-                .withSelector(selector).withTreatment(treatment)
                 .withPriority(100).withFlag(ForwardingObjective.Flag.SPECIFIC);
     }
 
@@ -454,7 +468,7 @@
                 if (srManager.mastershipService.isLocalMaster(deviceId)) {
                     TrafficTreatment tt = DefaultTrafficTreatment.builder()
                             .pushVlan().setVlanId(assignedVlan).build();
-                    fob.setMeta(tt);
+                    fob.withMeta(tt);
                 }
                 fob.permit().fromApp(srManager.appId);
                 srManager.flowObjectiveService.
@@ -559,6 +573,12 @@
             int nextId = srManager.getSubnetNextObjectiveId(deviceId, subnet);
             VlanId vlanId = srManager.getSubnetAssignedVlanId(deviceId, subnet);
 
+            if (nextId < 0 || vlanId == null) {
+                log.error("Cannot install subnet broadcast rule in dev:{} due"
+                        + "to vlanId:{} or nextId:{}", vlanId, nextId);
+                return;
+            }
+
             /* Driver should treat objective with MacAddress.NONE as the
              * subnet broadcast rule
              */
diff --git a/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java b/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
index 518bce3..62722f0 100644
--- a/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
+++ b/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
@@ -57,6 +57,7 @@
 import org.onosproject.segmentrouting.grouphandler.DefaultGroupHandler;
 import org.onosproject.segmentrouting.grouphandler.NeighborSet;
 import org.onosproject.segmentrouting.grouphandler.NeighborSetNextObjectiveStoreKey;
+import org.onosproject.segmentrouting.grouphandler.PortNextObjectiveStoreKey;
 import org.onosproject.mastership.MastershipService;
 import org.onosproject.net.Device;
 import org.onosproject.net.DeviceId;
@@ -97,7 +98,6 @@
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
-@SuppressWarnings("ALL")
 @Service
 @Component(immediate = true)
 public class SegmentRoutingManager implements SegmentRoutingService {
@@ -150,21 +150,27 @@
     private ScheduledExecutorService executorService = Executors
             .newScheduledThreadPool(1);
 
+    @SuppressWarnings("unused")
     private static ScheduledFuture<?> eventHandlerFuture = null;
+    @SuppressWarnings("rawtypes")
     private ConcurrentLinkedQueue<Event> eventQueue = new ConcurrentLinkedQueue<Event>();
     private Map<DeviceId, DefaultGroupHandler> groupHandlerMap =
             new ConcurrentHashMap<DeviceId, DefaultGroupHandler>();
     // Per device next objective ID store with (device id + neighbor set) as key
     private EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey, Integer>
             nsNextObjStore = null;
+    // Per device next objective ID store with (device id + subnet) as key
     private EventuallyConsistentMap<SubnetNextObjectiveStoreKey, Integer>
             subnetNextObjStore = null;
-    private EventuallyConsistentMap<String, Tunnel> tunnelStore = null;
-    private EventuallyConsistentMap<String, Policy> policyStore = null;
+    // Per device next objective ID store with (device id + port) as key
+    private EventuallyConsistentMap<PortNextObjectiveStoreKey, Integer>
+            portNextObjStore = null;
     // Per device, per-subnet assigned-vlans store, with (device id + subnet
     // IPv4 prefix) as key
     private EventuallyConsistentMap<SubnetAssignedVidStoreKey, VlanId>
         subnetVidStore = null;
+    private EventuallyConsistentMap<String, Tunnel> tunnelStore = null;
+    private EventuallyConsistentMap<String, Policy> policyStore = null;
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected StorageService storageService;
@@ -175,6 +181,7 @@
     private final InternalConfigListener cfgListener =
             new InternalConfigListener(this);
 
+    @SuppressWarnings({ "unchecked", "rawtypes" })
     private final ConfigFactory cfgFactory =
             new ConfigFactory(SubjectFactories.DEVICE_SUBJECT_FACTORY,
                               SegmentRoutingConfig.class,
@@ -228,7 +235,6 @@
         log.debug("Creating EC map nsnextobjectivestore");
         EventuallyConsistentMapBuilder<NeighborSetNextObjectiveStoreKey, Integer>
                 nsNextObjMapBuilder = storageService.eventuallyConsistentMapBuilder();
-
         nsNextObjStore = nsNextObjMapBuilder
                 .withName("nsnextobjectivestore")
                 .withSerializer(kryoBuilder)
@@ -239,16 +245,23 @@
         log.debug("Creating EC map subnetnextobjectivestore");
         EventuallyConsistentMapBuilder<SubnetNextObjectiveStoreKey, Integer>
                 subnetNextObjMapBuilder = storageService.eventuallyConsistentMapBuilder();
-
         subnetNextObjStore = subnetNextObjMapBuilder
                 .withName("subnetnextobjectivestore")
                 .withSerializer(kryoBuilder)
                 .withTimestampProvider((k, v) -> new WallClockTimestamp())
                 .build();
 
+        log.debug("Creating EC map subnetnextobjectivestore");
+        EventuallyConsistentMapBuilder<PortNextObjectiveStoreKey, Integer>
+                portNextObjMapBuilder = storageService.eventuallyConsistentMapBuilder();
+        portNextObjStore = portNextObjMapBuilder
+                .withName("portnextobjectivestore")
+                .withSerializer(kryoBuilder)
+                .withTimestampProvider((k, v) -> new WallClockTimestamp())
+                .build();
+
         EventuallyConsistentMapBuilder<String, Tunnel> tunnelMapBuilder =
                 storageService.eventuallyConsistentMapBuilder();
-
         tunnelStore = tunnelMapBuilder
                 .withName("tunnelstore")
                 .withSerializer(kryoBuilder)
@@ -257,7 +270,6 @@
 
         EventuallyConsistentMapBuilder<String, Policy> policyMapBuilder =
                 storageService.eventuallyConsistentMapBuilder();
-
         policyStore = policyMapBuilder
                 .withName("policystore")
                 .withSerializer(kryoBuilder)
@@ -266,7 +278,6 @@
 
         EventuallyConsistentMapBuilder<SubnetAssignedVidStoreKey, VlanId>
             subnetVidStoreMapBuilder = storageService.eventuallyConsistentMapBuilder();
-
         subnetVidStore = subnetVidStoreMapBuilder
                 .withName("subnetvidstore")
                 .withSerializer(kryoBuilder)
@@ -425,8 +436,7 @@
     /**
      * Returns the next objective ID for the given NeighborSet.
      * If the nextObjective does not exist, a new one is created and
-     * it's id is returned.
-     * TODO move the side-effect creation of a Next Objective into a new method
+     * its id is returned.
      *
      * @param deviceId Device ID
      * @param ns NegighborSet
@@ -441,18 +451,19 @@
             return groupHandlerMap
                     .get(deviceId).getNextObjectiveId(ns, meta);
         } else {
-            log.warn("getNextObjectiveId query in device {} not found", deviceId);
+            log.warn("getNextObjectiveId query - groupHandler for device {} "
+                    + "not found", deviceId);
             return -1;
         }
     }
 
     /**
-     * Returns the next objective ID for the Subnet given. If the nextObjectiveID does not exist,
-     * a new one is created and returned.
+     * Returns the next objective ID for the given subnet prefix. It is expected
+     * that the next-objective has been pre-created from configuration.
      *
      * @param deviceId Device ID
      * @param prefix Subnet
-     * @return next objective ID
+     * @return next objective ID or -1 if it was not found
      */
     public int getSubnetNextObjectiveId(DeviceId deviceId, IpPrefix prefix) {
         if (groupHandlerMap.get(deviceId) != null) {
@@ -460,7 +471,33 @@
             return groupHandlerMap
                     .get(deviceId).getSubnetNextObjectiveId(prefix);
         } else {
-            log.warn("getSubnetNextObjectiveId query in device {} not found", deviceId);
+            log.warn("getSubnetNextObjectiveId query - groupHandler for "
+                    + "device {} not found", deviceId);
+            return -1;
+        }
+    }
+
+    /**
+     * Returns the next objective ID for the given portNumber, given the treatment.
+     * There could be multiple different treatments to the same outport, which
+     * would result in different objectives. If the next object
+     * does not exist, a new one is created and its id is returned.
+     *
+     * @param deviceId Device ID
+     * @param portNum port number on device for which NextObjective is queried
+     * @param treatment the actions to apply on the packets (should include outport)
+     * @param meta metadata passed into the creation of a Next Objective if necessary
+     * @return next objective ID or -1 if it was not found
+     */
+    public int getPortNextObjectiveId(DeviceId deviceId, PortNumber portNum,
+                                      TrafficTreatment treatment,
+                                      TrafficSelector meta) {
+        DefaultGroupHandler ghdlr = groupHandlerMap.get(deviceId);
+        if (ghdlr != null) {
+            return ghdlr.getPortNextObjectiveId(portNum, treatment, meta);
+        } else {
+            log.warn("getPortNextObjectiveId query -  groupHandler for device {}"
+                    + " not found", deviceId);
             return -1;
         }
     }
@@ -475,7 +512,7 @@
 
             InboundPacket pkt = context.inPacket();
             Ethernet ethernet = pkt.parsed();
-
+            log.trace("Rcvd pktin: {}", ethernet);
             if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
                 arpHandler.processPacketIn(pkt);
             } else if (ethernet.getEtherType() == Ethernet.TYPE_IPV4) {
@@ -517,6 +554,7 @@
         }
     }
 
+    @SuppressWarnings("rawtypes")
     private void scheduleEventHandlerIfNotScheduled(Event event) {
         synchronized (threadSchedulerLock) {
             eventQueue.add(event);
@@ -539,6 +577,7 @@
         public void run() {
             try {
                 while (true) {
+                    @SuppressWarnings("rawtypes")
                     Event event = null;
                     synchronized (threadSchedulerLock) {
                         if (!eventQueue.isEmpty()) {
@@ -647,7 +686,8 @@
                                            linkService,
                                            flowObjectiveService,
                                            nsNextObjStore,
-                                           subnetNextObjStore);
+                                           subnetNextObjStore,
+                                           portNextObjStore);
             } catch (DeviceConfigNotFoundException e) {
                 log.warn(e.getMessage() + " Aborting processDeviceAdded.");
                 return;
@@ -714,7 +754,8 @@
                                                    linkService,
                                                    flowObjectiveService,
                                                    nsNextObjStore,
-                                                   subnetNextObjStore);
+                                                   subnetNextObjStore,
+                                                   portNextObjStore);
                     } catch (DeviceConfigNotFoundException e) {
                         log.warn(e.getMessage() + " Aborting configureNetwork.");
                         return;
@@ -766,7 +807,7 @@
 
                 // Populate bridging table entry
                 ForwardingObjective.Builder fob =
-                        getForwardingObjectiveBuilder(mac, vlanId, port);
+                        getForwardingObjectiveBuilder(deviceId, mac, vlanId, port);
                 flowObjectiveService.forward(deviceId, fob.add(
                         new BridgingTableObjectiveContext(mac, vlanId)
                 ));
@@ -782,20 +823,37 @@
         }
 
         private ForwardingObjective.Builder getForwardingObjectiveBuilder(
-                MacAddress mac, VlanId vlanId, PortNumber port) {
+                     DeviceId deviceId, MacAddress mac, VlanId vlanId,
+                     PortNumber outport) {
+            // match rule
             TrafficSelector.Builder sbuilder = DefaultTrafficSelector.builder();
             sbuilder.matchEthDst(mac);
             sbuilder.matchVlanId(vlanId);
 
             TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
-            // TODO Move popVlan from flow action to group action
             tbuilder.immediate().popVlan();
-            tbuilder.immediate().setOutput(port);
+            tbuilder.immediate().setOutput(outport);
+
+            // for switch pipelines that need it, provide outgoing vlan as metadata
+            VlanId outvlan = null;
+            Ip4Prefix subnet = deviceConfiguration.getPortSubnet(deviceId, outport);
+            if (subnet == null) {
+                outvlan = VlanId.vlanId(ASSIGNED_VLAN_NO_SUBNET);
+            } else {
+                outvlan = getSubnetAssignedVlanId(deviceId, subnet);
+            }
+            TrafficSelector meta = DefaultTrafficSelector.builder()
+                                        .matchVlanId(outvlan).build();
+
+            // All forwarding is via Groups. Drivers can re-purpose to flow-actions if needed.
+            int portNextObjId = getPortNextObjectiveId(deviceId, outport,
+                                                       tbuilder.build(),
+                                                       meta);
 
             return DefaultForwardingObjective.builder()
                     .withFlag(ForwardingObjective.Flag.SPECIFIC)
                     .withSelector(sbuilder.build())
-                    .withTreatment(tbuilder.build())
+                    .nextStep(portNextObjId)
                     .withPriority(100)
                     .fromApp(appId)
                     .makePermanent();
@@ -807,11 +865,13 @@
             DeviceId deviceId = event.subject().location().deviceId();
             PortNumber port = event.subject().location().port();
             Set<IpAddress> ips = event.subject().ipAddresses();
-            log.debug("Host {}/{} is added at {}:{}", mac, vlanId, deviceId, port);
+            log.info("Host {}/{} is added at {}:{}", mac, vlanId, deviceId, port);
 
             // Populate bridging table entry
+            log.debug("Populate L2 table entry for host {} at {}:{}",
+                      mac, deviceId, port);
             ForwardingObjective.Builder fob =
-                    getForwardingObjectiveBuilder(mac, vlanId, port);
+                    getForwardingObjectiveBuilder(deviceId, mac, vlanId, port);
             flowObjectiveService.forward(deviceId, fob.add(
                     new BridgingTableObjectiveContext(mac, vlanId)
             ));
@@ -835,7 +895,7 @@
 
             // Revoke bridging table entry
             ForwardingObjective.Builder fob =
-                    getForwardingObjectiveBuilder(mac, vlanId, port);
+                    getForwardingObjectiveBuilder(deviceId, mac, vlanId, port);
             flowObjectiveService.forward(deviceId, fob.remove(
                     new BridgingTableObjectiveContext(mac, vlanId)
             ));
@@ -863,7 +923,7 @@
 
             // Revoke previous bridging table entry
             ForwardingObjective.Builder prevFob =
-                    getForwardingObjectiveBuilder(mac, vlanId, prevPort);
+                    getForwardingObjectiveBuilder(prevDeviceId, mac, vlanId, prevPort);
             flowObjectiveService.forward(prevDeviceId, prevFob.remove(
                     new BridgingTableObjectiveContext(mac, vlanId)
             ));
@@ -878,7 +938,7 @@
 
             // Populate new bridging table entry
             ForwardingObjective.Builder newFob =
-                    getForwardingObjectiveBuilder(mac, vlanId, prevPort);
+                    getForwardingObjectiveBuilder(newDeviceId, mac, vlanId, newPort);
             flowObjectiveService.forward(newDeviceId, newFob.add(
                     new BridgingTableObjectiveContext(mac, vlanId)
             ));
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java
index 6b6d960..32c5365 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultEdgeGroupHandler.java
@@ -56,9 +56,11 @@
                                           NeighborSetNextObjectiveStoreKey,
                                           Integer> nsNextObjStore,
                                   EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
-                                          Integer> subnetNextObjStore) {
+                                          Integer> subnetNextObjStore,
+                                  EventuallyConsistentMap<PortNextObjectiveStoreKey,
+                                          Integer> portNextObjStore) {
         super(deviceId, appId, config, linkService, flowObjService,
-              nsNextObjStore, subnetNextObjStore);
+              nsNextObjStore, subnetNextObjStore, portNextObjStore);
     }
 
     @Override
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
index e792bf6..bc394b8 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
@@ -80,6 +80,8 @@
         NeighborSetNextObjectiveStoreKey, Integer> nsNextObjStore = null;
     protected EventuallyConsistentMap<
             SubnetNextObjectiveStoreKey, Integer> subnetNextObjStore = null;
+    protected EventuallyConsistentMap<
+            PortNextObjectiveStoreKey, Integer> portNextObjStore = null;
 
     protected KryoNamespace.Builder kryo = new KryoNamespace.Builder()
             .register(URI.class).register(HashSet.class)
@@ -93,11 +95,12 @@
                                   DeviceProperties config,
                                   LinkService linkService,
                                   FlowObjectiveService flowObjService,
-                                  EventuallyConsistentMap<
-                                          NeighborSetNextObjectiveStoreKey,
+                                  EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey,
                                           Integer> nsNextObjStore,
                                   EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
-                                          Integer> subnetNextObjStore) {
+                                          Integer> subnetNextObjStore,
+                                  EventuallyConsistentMap<PortNextObjectiveStoreKey,
+                                          Integer> portNextObjStore) {
         this.deviceId = checkNotNull(deviceId);
         this.appId = checkNotNull(appId);
         this.deviceConfig = checkNotNull(config);
@@ -114,6 +117,7 @@
         this.flowObjectiveService = flowObjService;
         this.nsNextObjStore = nsNextObjStore;
         this.subnetNextObjStore = subnetNextObjStore;
+        this.portNextObjStore = portNextObjStore;
 
         populateNeighborMaps();
     }
@@ -133,30 +137,34 @@
      * @throws DeviceConfigNotFoundException if the device configuration is not found
      * @return default group handler type
      */
-    public static DefaultGroupHandler createGroupHandler(DeviceId deviceId,
-                                                         ApplicationId appId,
-                                                         DeviceProperties config,
-                                                         LinkService linkService,
-                                                         FlowObjectiveService flowObjService,
-                                                         EventuallyConsistentMap<
-                                                                 NeighborSetNextObjectiveStoreKey,
-                                                                 Integer> nsNextObjStore,
-                                                         EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
-                                                                 Integer> subnetNextObjStore)
-                                                         throws DeviceConfigNotFoundException {
+    public static DefaultGroupHandler createGroupHandler(
+                                          DeviceId deviceId,
+                                          ApplicationId appId,
+                                          DeviceProperties config,
+                                          LinkService linkService,
+                                          FlowObjectiveService flowObjService,
+                                          EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey,
+                                          Integer> nsNextObjStore,
+                                          EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
+                                          Integer> subnetNextObjStore,
+                                          EventuallyConsistentMap<PortNextObjectiveStoreKey,
+                                          Integer> portNextObjStore)
+                                                  throws DeviceConfigNotFoundException {
         // handle possible exception in the caller
         if (config.isEdgeDevice(deviceId)) {
             return new DefaultEdgeGroupHandler(deviceId, appId, config,
                                                linkService,
                                                flowObjService,
                                                nsNextObjStore,
-                                               subnetNextObjStore);
+                                               subnetNextObjStore,
+                                               portNextObjStore);
         } else {
             return new DefaultTransitGroupHandler(deviceId, appId, config,
                                                   linkService,
                                                   flowObjService,
                                                   nsNextObjStore,
-                                                  subnetNextObjStore);
+                                                  subnetNextObjStore,
+                                                  portNextObjStore);
         }
     }
 
@@ -231,25 +239,21 @@
 
             Integer nextId = nsNextObjStore.
                     get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
-            if (nextId != null) {
+            if (nextId != null && isMaster) {
                 NextObjective.Builder nextObjBuilder = DefaultNextObjective
                         .builder().withId(nextId)
                         .withType(NextObjective.Type.HASHED).fromApp(appId);
 
                 nextObjBuilder.addTreatment(tBuilder.build());
-
                 log.info("**linkUp in device {}: Adding Bucket "
-                        + "with Port {} to next object id {} and amIMaster:{}",
+                        + "with Port {} to next object id {}",
                         deviceId,
                         newLink.src().port(),
-                        nextId, isMaster);
-
-                if (isMaster) {
-                    NextObjective nextObjective = nextObjBuilder.
-                            addToExisting(new SRNextObjectiveContext(deviceId));
-                    flowObjectiveService.next(deviceId, nextObjective);
-                }
-            } else {
+                        nextId);
+                NextObjective nextObjective = nextObjBuilder.
+                        addToExisting(new SRNextObjectiveContext(deviceId));
+                flowObjectiveService.next(deviceId, nextObjective);
+            } else if (isMaster) {
                 log.warn("linkUp in device {}, but global store has no record "
                         + "for neighbor-set {}", deviceId, ns);
             }
@@ -331,8 +335,8 @@
     }
 
     /**
-     * Returns the next objective associated with the neighborset.
-     * If there is no next objective for this neighborset, this API
+     * Returns the next objective of type hashed associated with the neighborset.
+     * If there is no next objective for this neighborset, this method
      * would create a next objective and return. Optionally metadata can be
      * passed in for the creation of the next objective.
      *
@@ -372,9 +376,10 @@
     }
 
     /**
-     * Returns the next objective associated with the subnet.
-     * If there is no next objective for this subnet, this API
-     * would create a next objective and return.
+     * Returns the next objective of type broadcast associated with the subnet,
+     * or -1 if no such objective exists. Note that this method does NOT create
+     * the next objective as a side-effect. It is expected that is objective is
+     * created at startup from network configuration.
      *
      * @param prefix subnet information
      * @return int if found or -1
@@ -387,6 +392,38 @@
     }
 
     /**
+     * Returns the next objective of type simple associated with the port on the
+     * device, given the treatment. Different treatments to the same port result
+     * in different next objectives. If no such objective exists, this method
+     * creates one and returns the id. Optionally metadata can be passed in for
+     * the creation of the objective.
+     *
+     * @param portNum the port number for the simple next objective
+     * @param treatment the actions to apply on the packets (should include outport)
+     * @param meta optional metadata passed into the creation of the next objective
+     * @return int if found or created, -1 if there are errors during the
+     *          creation of the next objective.
+     */
+    public int getPortNextObjectiveId(PortNumber portNum, TrafficTreatment treatment,
+                                      TrafficSelector meta) {
+        Integer nextId = portNextObjStore.
+                get(new PortNextObjectiveStoreKey(deviceId, portNum, treatment));
+        if (nextId == null) {
+            log.trace("getPortNextObjectiveId in device{}: Next objective id "
+                    + "not found for {} and {} creating", deviceId, portNum);
+            createGroupFromPort(portNum, treatment, meta);
+            nextId = portNextObjStore.get(
+                         new PortNextObjectiveStoreKey(deviceId, portNum, treatment));
+            if (nextId == null) {
+                log.warn("getPortNextObjectiveId: unable to create next obj"
+                        + "for dev:{} port{}", deviceId, portNum);
+                return -1;
+            }
+        }
+        return nextId;
+    }
+
+    /**
      * Checks if the next objective ID (group) for the neighbor set exists or not.
      *
      * @param ns neighbor set to check
@@ -561,7 +598,7 @@
                 }
             }
             if (meta != null) {
-                nextObjBuilder.setMeta(meta);
+                nextObjBuilder.withMeta(meta);
             }
             NextObjective nextObj = nextObjBuilder.
                     add(new SRNextObjectiveContext(deviceId));
@@ -574,7 +611,10 @@
         }
     }
 
-
+    /**
+     * Creates broadcast groups for all ports in the same configured subnet.
+     *
+     */
     public void createGroupsFromSubnetConfig() {
         Map<Ip4Prefix, List<PortNumber>> subnetPortMap =
                 this.deviceConfig.getSubnetPortsMap(this.deviceId);
@@ -612,6 +652,37 @@
         });
     }
 
+
+    /**
+     * Create simple next objective for a single port. The treatments can include
+     * all outgoing actions that need to happen on the packet.
+     *
+     * @param portNum  the outgoing port on the device
+     * @param treatment the actions to apply on the packets (should include outport)
+     * @param meta optional data to pass to the driver
+     */
+    public void createGroupFromPort(PortNumber portNum, TrafficTreatment treatment,
+                                    TrafficSelector meta) {
+        int nextId = flowObjectiveService.allocateNextId();
+        PortNextObjectiveStoreKey key = new PortNextObjectiveStoreKey(
+                                                deviceId, portNum, treatment);
+
+        NextObjective.Builder nextObjBuilder = DefaultNextObjective
+                .builder().withId(nextId)
+                .withType(NextObjective.Type.SIMPLE)
+                .addTreatment(treatment)
+                .fromApp(appId)
+                .withMeta(meta);
+
+        NextObjective nextObj = nextObjBuilder.add();
+        flowObjectiveService.next(deviceId, nextObj);
+        log.debug("createGroupFromPort: Submited next objective {} in device {} "
+                + "for port {}", nextId, deviceId, portNum);
+
+        portNextObjStore.put(key, nextId);
+    }
+
+
     public GroupKey getGroupKey(Object obj) {
         return new DefaultGroupKey(kryo.build().serialize(obj));
     }
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java
index 14d77ba..7a43e73 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultTransitGroupHandler.java
@@ -50,9 +50,11 @@
                                         NeighborSetNextObjectiveStoreKey,
                                         Integer> nsNextObjStore,
                                   EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
-                                        Integer> subnetNextObjStore) {
+                                        Integer> subnetNextObjStore,
+                                  EventuallyConsistentMap<PortNextObjectiveStoreKey,
+                                  Integer> portNextObjStore) {
         super(deviceId, appId, config, linkService, flowObjService,
-              nsNextObjStore, subnetNextObjStore);
+              nsNextObjStore, subnetNextObjStore, portNextObjStore);
     }
 
     @Override
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
index 5514207..ef143dc 100644
--- a/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/PolicyGroupHandler.java
@@ -68,9 +68,11 @@
                               EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey,
                                       Integer> nsNextObjStore,
                               EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
-                                      Integer> subnetNextObjStore) {
+                                      Integer> subnetNextObjStore,
+                              EventuallyConsistentMap<PortNextObjectiveStoreKey,
+                              Integer> portNextObjStore) {
         super(deviceId, appId, config, linkService, flowObjService,
-              nsNextObjStore, subnetNextObjStore);
+              nsNextObjStore, subnetNextObjStore, portNextObjStore);
     }
 
     public PolicyGroupIdentifier createPolicyGroupChain(String id,
diff --git a/src/main/java/org/onosproject/segmentrouting/grouphandler/PortNextObjectiveStoreKey.java b/src/main/java/org/onosproject/segmentrouting/grouphandler/PortNextObjectiveStoreKey.java
new file mode 100644
index 0000000..5555565
--- /dev/null
+++ b/src/main/java/org/onosproject/segmentrouting/grouphandler/PortNextObjectiveStoreKey.java
@@ -0,0 +1,77 @@
+package org.onosproject.segmentrouting.grouphandler;
+
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.flow.TrafficTreatment;
+
+import java.util.Objects;
+
+/**
+ * Class definition of Key for Device/Port to NextObjective store. Since there
+ * can be multiple next objectives to the same physical port, we differentiate
+ * between them by including the treatment in the key.
+ */
+public class PortNextObjectiveStoreKey {
+    private final DeviceId deviceId;
+    private final PortNumber portNum;
+    private final TrafficTreatment treatment;
+
+    public PortNextObjectiveStoreKey(DeviceId deviceId, PortNumber portNum,
+                                     TrafficTreatment treatment) {
+        this.deviceId = deviceId;
+        this.portNum = portNum;
+        this.treatment = treatment;
+    }
+
+    /**
+     * Gets device id in this PortNextObjectiveStoreKey.
+     *
+     * @return device id
+     */
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    /**
+     * Gets port information in this PortNextObjectiveStoreKey.
+     *
+     * @return port information
+     */
+    public PortNumber portNumber() {
+        return portNum;
+    }
+
+    /**
+     * Gets treatment information in this PortNextObjectiveStoreKey.
+     *
+     * @return treatment information
+     */
+    public TrafficTreatment treatment() {
+        return treatment;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof PortNextObjectiveStoreKey)) {
+            return false;
+        }
+        PortNextObjectiveStoreKey that =
+                (PortNextObjectiveStoreKey) o;
+        return (Objects.equals(this.deviceId, that.deviceId) &&
+                Objects.equals(this.portNum, that.portNum) &&
+                Objects.equals(this.treatment, that.treatment));
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(deviceId, portNum, treatment);
+    }
+
+    @Override
+    public String toString() {
+        return "Device: " + deviceId + " Port: " + portNum + " Treatment: " + treatment;
+    }
+}