Route reprogamming using group substitution during next hop movement

Change-Id: Idf8362dac522722ca67747e245bfd836e6ee6292
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
index a5a90a7..7dfa632 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/DefaultRoutingHandler.java
@@ -1118,11 +1118,12 @@
      * @param hostMac MAC address of the next hop
      * @param hostVlanId Vlan ID of the nexthop
      * @param outPort port where the next hop attaches to
+     * @param directHost host is of type direct or indirect
      */
     void populateRoute(DeviceId deviceId, IpPrefix prefix,
-                       MacAddress hostMac, VlanId hostVlanId, PortNumber outPort) {
+                       MacAddress hostMac, VlanId hostVlanId, PortNumber outPort, boolean directHost) {
         if (shouldProgram(deviceId)) {
-            srManager.routingRulePopulator.populateRoute(deviceId, prefix, hostMac, hostVlanId, outPort);
+            srManager.routingRulePopulator.populateRoute(deviceId, prefix, hostMac, hostVlanId, outPort, directHost);
         }
     }
 
@@ -1135,11 +1136,12 @@
      * @param hostMac MAC address of the next hop
      * @param hostVlanId Vlan ID of the nexthop
      * @param outPort port that next hop attaches to
+     * @param directHost host is of type direct or indirect
      */
     void revokeRoute(DeviceId deviceId, IpPrefix prefix,
-                     MacAddress hostMac, VlanId hostVlanId, PortNumber outPort) {
+                     MacAddress hostMac, VlanId hostVlanId, PortNumber outPort, boolean directHost) {
         if (shouldProgram(deviceId)) {
-            srManager.routingRulePopulator.revokeRoute(deviceId, prefix, hostMac, hostVlanId, outPort);
+            srManager.routingRulePopulator.revokeRoute(deviceId, prefix, hostMac, hostVlanId, outPort, directHost);
         }
     }
 
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/HostHandler.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/HostHandler.java
index eee9900..296fbcd 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/HostHandler.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/HostHandler.java
@@ -525,9 +525,9 @@
 
         log.info("{} routing rule for {} at {}", revoke ? "Revoking" : "Populating", ip, location);
         if (revoke) {
-            srManager.defaultRoutingHandler.revokeRoute(deviceId, ip.toIpPrefix(), mac, vlanId, port);
+            srManager.defaultRoutingHandler.revokeRoute(deviceId, ip.toIpPrefix(), mac, vlanId, port, true);
         } else {
-            srManager.defaultRoutingHandler.populateRoute(deviceId, ip.toIpPrefix(), mac, vlanId, port);
+            srManager.defaultRoutingHandler.populateRoute(deviceId, ip.toIpPrefix(), mac, vlanId, port, true);
         }
     }
 
@@ -675,10 +675,10 @@
                 ipPrefixSet.forEach(ipPrefix -> {
                     if (install && ipPrefix.contains(hostIpAddress)) {
                             srManager.defaultRoutingHandler.populateRoute(cp.deviceId(), hostIpAddress.toIpPrefix(),
-                                                                         host.mac(), host.vlan(), cp.port());
+                                                                         host.mac(), host.vlan(), cp.port(), true);
                     } else if (!install && ipPrefix.contains(hostIpAddress)) {
                             srManager.defaultRoutingHandler.revokeRoute(cp.deviceId(), hostIpAddress.toIpPrefix(),
-                                                                       host.mac(), host.vlan(), cp.port());
+                                                                       host.mac(), host.vlan(), cp.port(), true);
                     }
                 });
             }));
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/RouteHandler.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/RouteHandler.java
index 40cdf53..7dbbbeb 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/RouteHandler.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/RouteHandler.java
@@ -32,6 +32,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Collection;
 import java.util.Objects;
 import java.util.Optional;
@@ -101,7 +103,7 @@
                 srManager.deviceConfiguration.addSubnet(location, prefix);
                 log.debug("RouteAdded populateRoute {}, {}, {}, {}", location, prefix, nextHopMac, nextHopVlan);
                 srManager.defaultRoutingHandler.populateRoute(location.deviceId(), prefix,
-                        nextHopMac, nextHopVlan, location.port());
+                        nextHopMac, nextHopVlan, location.port(), false);
             });
         });
     }
@@ -175,7 +177,7 @@
                 srManager.deviceConfiguration.addSubnet(location, prefix);
                 log.debug("RouteUpdated. populateRoute {}, {}, {}, {}", location, prefix, nextHopMac, nextHopVlan);
                 srManager.defaultRoutingHandler.populateRoute(location.deviceId(), prefix,
-                        nextHopMac, nextHopVlan, location.port());
+                        nextHopMac, nextHopVlan, location.port(), false);
             });
         });
     }
@@ -221,7 +223,7 @@
 
                     log.debug("RouteRemoved. revokeRoute {}, {}, {}, {}", location, prefix, nextHopMac, nextHopVlan);
                     srManager.defaultRoutingHandler.revokeRoute(pairDeviceId.get(), prefix,
-                            nextHopMac, vlanId, pairLocalPort.get());
+                            nextHopMac, vlanId, pairLocalPort.get(), false);
                 }
             });
         });
@@ -231,6 +233,8 @@
         log.info("processHostMovedEvent {}", event);
         MacAddress hostMac = event.subject().mac();
         VlanId hostVlanId = event.subject().vlan();
+        // map of nextId for prev port in each device
+        Map<DeviceId, Integer> nextIdMap = new HashMap<>();
 
         affectedRoutes(hostMac, hostVlanId).forEach(affectedRoute -> {
             IpPrefix prefix = affectedRoute.prefix();
@@ -245,8 +249,22 @@
             Set<DeviceId> newDeviceIds = newLocations.stream().map(HostLocation::deviceId)
                     .collect(Collectors.toSet());
 
+            // Set of deviceIDs of the previous locations where the host was connected
+            // Used to determine if host moved to different connect points
+            // on same device or moved to a different device altogether
+            Set<DeviceId> oldDeviceIds = prevLocations.stream().map(HostLocation::deviceId)
+                    .collect(Collectors.toSet());
+
             // For each old location
             Sets.difference(prevLocations, newLocations).forEach(prevLocation -> {
+                //find next Id for each old port, add to map
+                int nextId = srManager.getNextIdForHostPort(prevLocation.deviceId(), hostMac,
+                                                             hostVlanId, prevLocation.port());
+                log.debug("HostMoved. NextId For Host {}, {}, {}, {} {}", prevLocation, prefix,
+                                                             hostMac, hostVlanId, nextId);
+
+                nextIdMap.put(prevLocation.deviceId(), nextId);
+
                 // Remove flows for unchanged IPs only when the host moves from a switch to another.
                 // Otherwise, do not remove and let the adding part update the old flow
                 if (newDeviceIds.contains(prevLocation.deviceId())) {
@@ -265,7 +283,7 @@
 
                 log.debug("HostMoved. revokeRoute {}, {}, {}, {}", prevLocation, prefix, hostMac, hostVlanId);
                 srManager.defaultRoutingHandler.revokeRoute(prevLocation.deviceId(), prefix,
-                        hostMac, hostVlanId, prevLocation.port());
+                        hostMac, hostVlanId, prevLocation.port(), false);
             });
 
             // For each new location, add all new IPs.
@@ -273,9 +291,20 @@
                 log.debug("HostMoved. addSubnet {}, {}", newLocation, prefix);
                 srManager.deviceConfiguration.addSubnet(newLocation, prefix);
 
-                log.debug("HostMoved. populateRoute {}, {}, {}, {}", newLocation, prefix, hostMac, hostVlanId);
-                srManager.defaultRoutingHandler.populateRoute(newLocation.deviceId(), prefix,
-                        hostMac, hostVlanId, newLocation.port());
+                //its a new connect point, not a move from an existing device, populateRoute
+                if (!oldDeviceIds.contains(newLocation.deviceId())) {
+                   log.debug("HostMoved. populateRoute {}, {}, {}, {}", newLocation, prefix, hostMac, hostVlanId);
+                   srManager.defaultRoutingHandler.populateRoute(newLocation.deviceId(), prefix,
+                          hostMac, hostVlanId, newLocation.port(), false);
+                } else {
+                  // update same flow to point to new nextObj
+                  VlanId vlanId = Optional.ofNullable(srManager.getInternalVlanId(newLocation)).orElse(hostVlanId);
+                  //use nextIdMap to send new port details to update the nextId group bucket
+                  log.debug("HostMoved. update L3 Ucast Group Bucket {}, {}, {} --> {}",
+                                         newLocation, hostMac, vlanId, nextIdMap.get(newLocation.deviceId()));
+                  srManager.updateMacVlanTreatment(newLocation.deviceId(), hostMac, vlanId,
+                          newLocation.port(), nextIdMap.get(newLocation.deviceId()));
+                }
             });
 
         });
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
index 768c715..bddf2d2 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/RoutingRulePopulator.java
@@ -307,15 +307,16 @@
      * @param hostMac MAC address of the next hop
      * @param hostVlanId Vlan ID of the nexthop
      * @param outPort port where the next hop attaches to
+     * @param directHost host is of type direct or indirect
      */
     void populateRoute(DeviceId deviceId, IpPrefix prefix,
-                              MacAddress hostMac, VlanId hostVlanId, PortNumber outPort) {
+                              MacAddress hostMac, VlanId hostVlanId, PortNumber outPort, boolean directHost) {
         log.debug("Populate direct routing entry for route {} at {}:{}",
                 prefix, deviceId, outPort);
         ForwardingObjective.Builder fwdBuilder;
         try {
             fwdBuilder = routingFwdObjBuilder(deviceId, prefix, hostMac,
-                                              hostVlanId, outPort, false);
+                                              hostVlanId, outPort, directHost, false);
         } catch (DeviceConfigNotFoundException e) {
             log.warn(e.getMessage() + " Aborting direct populateRoute");
             return;
@@ -346,15 +347,16 @@
      * @param hostMac MAC address of the next hop
      * @param hostVlanId Vlan ID of the nexthop
      * @param outPort port that next hop attaches to
+     * @param directHost host is of type direct or indirect
      */
     void revokeRoute(DeviceId deviceId, IpPrefix prefix,
-            MacAddress hostMac, VlanId hostVlanId, PortNumber outPort) {
+            MacAddress hostMac, VlanId hostVlanId, PortNumber outPort, boolean directHost) {
         log.debug("Revoke IP table entry for route {} at {}:{}",
                 prefix, deviceId, outPort);
         ForwardingObjective.Builder fwdBuilder;
         try {
             fwdBuilder = routingFwdObjBuilder(deviceId, prefix, hostMac,
-                                              hostVlanId, outPort, true);
+                                              hostVlanId, outPort, directHost, true);
         } catch (DeviceConfigNotFoundException e) {
             log.warn(e.getMessage() + " Aborting revokeIpRuleForHost.");
             return;
@@ -383,16 +385,19 @@
      * @param hostVlanId Vlan ID of the nexthop
      * @param outPort port where the nexthop attaches to
      * @param revoke true if forwarding objective is meant to revoke forwarding rule
+     * @param directHost host is direct or indirect
      * @return forwarding objective builder
      * @throws DeviceConfigNotFoundException if given device is not configured
      */
+
     private ForwardingObjective.Builder routingFwdObjBuilder(
             DeviceId deviceId, IpPrefix prefix,
             MacAddress hostMac, VlanId hostVlanId, PortNumber outPort,
-            boolean revoke)
+            boolean directHost, boolean revoke)
             throws DeviceConfigNotFoundException {
         MacAddress deviceMac;
         deviceMac = config.getDeviceMac(deviceId);
+        int nextObjId = -1;
 
         ConnectPoint connectPoint = new ConnectPoint(deviceId, outPort);
         VlanId untaggedVlan = srManager.interfaceService.getUntaggedVlanId(connectPoint);
@@ -439,18 +444,26 @@
                 return null;
             }
         }
-        // if the objective is to revoke an existing rule, and for some reason
-        // the next-objective does not exist, then a new one should not be created
-        int portNextObjId = srManager.getPortNextObjectiveId(deviceId, outPort,
+
+        if (directHost) {
+          // if the objective is to revoke an existing rule, and for some reason
+          // the next-objective does not exist, then a new one should not be created
+          nextObjId = srManager.getPortNextObjectiveId(deviceId, outPort,
                                           tbuilder.build(), mbuilder.build(), !revoke);
-        if (portNextObjId == -1) {
-            // Warning log will come from getPortNextObjective method
+        } else {
+          // if the objective is to revoke an existing rule, and for some reason
+          // the next-objective does not exist, then a new one should not be created
+          nextObjId = srManager.getMacVlanNextObjectiveId(deviceId, hostMac, hostVlanId,
+                                          tbuilder.build(), mbuilder.build(), !revoke);
+        }
+        if (nextObjId == -1) {
+            // Warning log will come from getMacVlanNextObjective method
             return null;
         }
 
         return DefaultForwardingObjective.builder()
                 .withSelector(sbuilder.build())
-                .nextStep(portNextObjId)
+                .nextStep(nextObjId)
                 .fromApp(srManager.appId).makePermanent()
                 .withPriority(getPriorityFromPrefix(prefix))
                 .withFlag(ForwardingObjective.Flag.SPECIFIC);
@@ -1725,7 +1738,7 @@
                     ));
                 });
         try {
-            fwdBuilder = routingFwdObjBuilder(deviceId, prefix, hostMac, dummyVlan, outPort, false);
+            fwdBuilder = routingFwdObjBuilder(deviceId, prefix, hostMac, dummyVlan, outPort, false, false);
         } catch (DeviceConfigNotFoundException e) {
             log.error(e.getMessage() + " Aborting populateDoubleTaggedRoute");
             return;
@@ -1764,7 +1777,7 @@
     void revokeDoubleTaggedRoute(DeviceId deviceId, IpPrefix prefix, MacAddress hostMac,
                                  VlanId hostVlan, VlanId innerVlan, VlanId outerVlan,
                                  EthType outerTpid, PortNumber outPort) {
-        revokeRoute(deviceId, prefix, hostMac, hostVlan, outPort);
+        revokeRoute(deviceId, prefix, hostMac, hostVlan, outPort, false);
 
         DummyVlanIdStoreKey key = new DummyVlanIdStoreKey(
                 new ConnectPoint(deviceId, outPort), prefix.address());
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
index 07f3ecf..422be0f 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
@@ -36,6 +36,7 @@
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.VlanId;
+import org.onlab.packet.MacAddress;
 import org.onlab.util.KryoNamespace;
 import org.onlab.util.Tools;
 import org.onosproject.cfg.ComponentConfigService;
@@ -126,6 +127,7 @@
 import org.onosproject.segmentrouting.mcast.McastStoreKey;
 import org.onosproject.segmentrouting.storekey.PortNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.VlanNextObjectiveStoreKey;
+import org.onosproject.segmentrouting.storekey.MacVlanNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.XConnectStoreKey;
 import org.onosproject.segmentrouting.xconnect.api.XconnectService;
 import org.onosproject.store.serializers.KryoNamespaces;
@@ -333,7 +335,7 @@
             vlanNextObjStore = null;
     /**
      * Per device next objective ID store with (device id + port + treatment + meta) as key.
-     * Used to keep track on L2 interface group and L3 unicast group information.
+     * Used to keep track on L2 interface group and L3 unicast group information for direct hosts.
      */
     private EventuallyConsistentMap<PortNextObjectiveStoreKey, Integer>
             portNextObjStore = null;
@@ -345,6 +347,13 @@
     private EventuallyConsistentMap<DummyVlanIdStoreKey, VlanId>
             dummyVlanIdStore = null;
 
+    /**
+     * Per device next objective ID store with (device id + MAC address + vlan) as key.
+     * Used to keep track of L3 unicast group for indirect hosts.
+     */
+    private EventuallyConsistentMap<MacVlanNextObjectiveStoreKey, Integer>
+            macVlanNextObjStore = null;
+
     private EventuallyConsistentMap<String, Tunnel> tunnelStore = null;
     private EventuallyConsistentMap<String, Policy> policyStore = null;
 
@@ -458,6 +467,15 @@
                 .withTimestampProvider((k, v) -> new WallClockTimestamp())
                 .build();
 
+        log.debug("Creating EC map macvlannextobjectivestore");
+        EventuallyConsistentMapBuilder<MacVlanNextObjectiveStoreKey, Integer>
+                macVlanNextObjMapBuilder = storageService.eventuallyConsistentMapBuilder();
+        macVlanNextObjStore = macVlanNextObjMapBuilder
+                .withName("macvlannextobjectivestore")
+                .withSerializer(createSerializer())
+                .withTimestampProvider((k, v) -> new WallClockTimestamp())
+                .build();
+
         log.debug("Creating EC map subnetnextobjectivestore");
         EventuallyConsistentMapBuilder<PortNextObjectiveStoreKey, Integer>
                 portNextObjMapBuilder = storageService.eventuallyConsistentMapBuilder();
@@ -584,7 +602,8 @@
                           L2TunnelPolicy.class,
                           DefaultL2Tunnel.class,
                           DefaultL2TunnelPolicy.class,
-                          DummyVlanIdStoreKey.class
+                          DummyVlanIdStoreKey.class,
+                          MacVlanNextObjectiveStoreKey.class
                 );
     }
 
@@ -632,6 +651,7 @@
 
         dsNextObjStore.destroy();
         vlanNextObjStore.destroy();
+        macVlanNextObjStore.destroy();
         portNextObjStore.destroy();
         dummyVlanIdStore.destroy();
         tunnelStore.destroy();
@@ -928,6 +948,15 @@
     }
 
     @Override
+    public ImmutableMap<MacVlanNextObjectiveStoreKey, Integer> getMacVlanNextObjStore() {
+        if (macVlanNextObjStore != null) {
+            return ImmutableMap.copyOf(macVlanNextObjStore.entrySet());
+        } else {
+            return ImmutableMap.of();
+        }
+    }
+
+    @Override
     public ImmutableMap<PortNextObjectiveStoreKey, Integer> getPortNextObjStore() {
         if (portNextObjStore != null) {
             return ImmutableMap.copyOf(portNextObjStore.entrySet());
@@ -970,6 +999,13 @@
                 }
             });
         }
+        if (macVlanNextObjStore != null) {
+            macVlanNextObjStore.entrySet().forEach(e -> {
+                if (e.getValue() == nextId) {
+                    macVlanNextObjStore.remove(e.getKey());
+                }
+            });
+        }
         if (portNextObjStore != null) {
             portNextObjStore.entrySet().forEach(e -> {
                 if (e.getValue() == nextId) {
@@ -1090,8 +1126,18 @@
     }
 
     /**
+     * Per device next objective ID store with (device id + MAC address + vlan) as key.
+     * Used to keep track on L3 Unicast group information for indirect hosts.
+     *
+     * @return mac vlan next object store
+     */
+    public EventuallyConsistentMap<MacVlanNextObjectiveStoreKey, Integer> macVlanNextObjStore() {
+        return macVlanNextObjStore;
+    }
+
+    /**
      * Per device next objective ID store with (device id + port + treatment + meta) as key.
-     * Used to keep track on L2 interface group and L3 unicast group information.
+     * Used to keep track on L2 interface group and L3 unicast group information for direct hosts.
      *
      * @return port next object store.
      */
@@ -1235,6 +1281,76 @@
     }
 
     /**
+     * Returns the next Objective ID for the given mac and vlan, given the treatment.
+     * There could be multiple different treatments to the same outport, which
+     * would result in different objectives. If the next object does not exist,
+     * and should be created, a new one is created and its id is returned.
+     *
+     * @param deviceId Device ID
+     * @param macAddr mac of host for which Next ID is required.
+     * @param vlanId vlan of host for which Next ID is required.
+     * @param treatment the actions to apply on the packets (should include outport)
+     * @param meta metadata passed into the creation of a Next Objective if necessary
+     * @param createIfMissing true if a next object should be created if not found
+     * @return next objective ID or -1 if an error occurred during retrieval or creation
+     */
+    public int getMacVlanNextObjectiveId(DeviceId deviceId, MacAddress macAddr, VlanId vlanId,
+                                      TrafficTreatment treatment,
+                                      TrafficSelector meta,
+                                      boolean createIfMissing) {
+        DefaultGroupHandler ghdlr = groupHandlerMap.get(deviceId);
+        if (ghdlr != null) {
+            return ghdlr.getMacVlanNextObjectiveId(macAddr, vlanId, treatment, meta, createIfMissing);
+        } else {
+            log.warn("getMacVlanNextObjectiveId query - groupHandler for device {}"
+                    + " not found", deviceId);
+            return -1;
+        }
+    }
+
+    /**
+     * Returns the next ID for the given host port from the store.
+     *
+     * @param deviceId Device ID
+     * @param hostMac mac of host for which Next ID is required.
+     * @param hostVlanId vlan of host for which Next ID is required.
+     * @param port port of device for which next ID is required.
+     * @return next objective ID or -1 if an error occurred during retrieval
+     */
+    public int getNextIdForHostPort(DeviceId deviceId, MacAddress hostMac,
+                                     VlanId hostVlanId, PortNumber port) {
+        DefaultGroupHandler ghdlr = groupHandlerMap.get(deviceId);
+        if (ghdlr != null) {
+            return ghdlr.getNextIdForHostPort(hostMac, hostVlanId, port);
+        } else {
+            log.warn("getNextIdForHostPort query - groupHandler for device {}"
+                    + " not found", deviceId);
+            return -1;
+        }
+    }
+
+    /**
+     * Updates the next objective for the given nextId .
+     *
+     * @param deviceId Device ID
+     * @param hostMac mac of host for which Next obj is to be updated.
+     * @param hostVlanId vlan of host for which Next obj is to be updated.
+     * @param port port with which to update the Next Obj.
+     * @param nextId of Next Obj which needs to be updated.
+     */
+    public void updateMacVlanTreatment(DeviceId deviceId, MacAddress hostMac,
+                             VlanId hostVlanId, PortNumber port, int nextId) {
+        DefaultGroupHandler ghdlr = groupHandlerMap.get(deviceId);
+        if (ghdlr != null) {
+            ghdlr.updateL3UcastGroupBucket(hostMac, hostVlanId, port, nextId);
+        } else {
+            log.warn("updateL3UcastGroupBucket query - groupHandler for device {}"
+                    + " not found", deviceId);
+        }
+    }
+
+
+    /**
      * Returns the group handler object for the specified device id.
      *
      * @param devId the device identifier
@@ -1558,6 +1674,9 @@
         vlanNextObjStore.entrySet().stream()
                 .filter(entry -> entry.getKey().deviceId().equals(device.id()))
                 .forEach(entry -> vlanNextObjStore.remove(entry.getKey()));
+        macVlanNextObjStore.entrySet().stream()
+                .filter(entry -> entry.getKey().deviceId().equals(device.id()))
+                .forEach(entry -> macVlanNextObjStore.remove(entry.getKey()));
         portNextObjStore.entrySet().stream()
                 .filter(entry -> entry.getKey().deviceId().equals(device.id()))
                 .forEach(entry -> portNextObjStore.remove(entry.getKey()));
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingService.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingService.java
index 1b28998..74db140 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingService.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingService.java
@@ -42,6 +42,7 @@
 import org.onosproject.segmentrouting.mcast.McastStoreKey;
 import org.onosproject.segmentrouting.storekey.PortNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.VlanNextObjectiveStoreKey;
+import org.onosproject.segmentrouting.storekey.MacVlanNextObjectiveStoreKey;
 
 import java.util.List;
 import java.util.Map;
@@ -226,6 +227,13 @@
     ImmutableMap<VlanNextObjectiveStoreKey, Integer> getVlanNextObjStore();
 
     /**
+     * Returns the Mac Vlan next objective store.
+     *
+     * @return current contents of the macVlanNextObjStore
+     */
+    ImmutableMap<MacVlanNextObjectiveStoreKey, Integer> getMacVlanNextObjStore();
+
+    /**
      * Returns the port next objective store.
      *
      * @return current contents of the portNextObjStore
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/cli/NextMacVlanCommand.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/cli/NextMacVlanCommand.java
new file mode 100644
index 0000000..e952915
--- /dev/null
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/cli/NextMacVlanCommand.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.segmentrouting.cli;
+
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.segmentrouting.SegmentRoutingService;
+import org.onosproject.segmentrouting.storekey.MacVlanNextObjectiveStoreKey;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Map;
+
+/**
+ * Command to read the current state of the macVlanNextObjStore.
+ */
+@Command(scope = "onos", name = "sr-next-mac-vlan",
+        description = "Displays the current next-hop / next-id it mapping")
+public class NextMacVlanCommand extends AbstractShellCommand {
+    @Override
+    protected void execute() {
+        SegmentRoutingService srService =
+                AbstractShellCommand.get(SegmentRoutingService.class);
+        print(srService.getMacVlanNextObjStore());
+    }
+
+    private void print(Map<MacVlanNextObjectiveStoreKey, Integer> macVlanNextObjStore) {
+        ArrayList<MacVlanNextObjectiveStoreKey> a = new ArrayList<>(macVlanNextObjStore.keySet());
+        a.sort(Comparator
+                .comparing((MacVlanNextObjectiveStoreKey o) -> o.deviceId().toString())
+                .thenComparing((MacVlanNextObjectiveStoreKey o) -> o.vlanId().toString())
+                .thenComparing((MacVlanNextObjectiveStoreKey o) -> o.macAddr().toString()));
+
+        StringBuilder builder = new StringBuilder();
+        a.forEach(k ->
+            builder.append("\n")
+                    .append(k)
+                    .append(" --> ")
+                    .append(macVlanNextObjStore.get(k))
+        );
+        print(builder.toString());
+    }
+}
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
index 1726fe7..716178d 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/grouphandler/DefaultGroupHandler.java
@@ -44,9 +44,11 @@
 import org.onosproject.segmentrouting.SegmentRoutingManager;
 import org.onosproject.segmentrouting.config.DeviceConfigNotFoundException;
 import org.onosproject.segmentrouting.config.DeviceProperties;
+import org.onosproject.segmentrouting.config.DeviceConfiguration;
 import org.onosproject.segmentrouting.storekey.DestinationSetNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.PortNextObjectiveStoreKey;
 import org.onosproject.segmentrouting.storekey.VlanNextObjectiveStoreKey;
+import org.onosproject.segmentrouting.storekey.MacVlanNextObjectiveStoreKey;
 import org.onosproject.store.service.EventuallyConsistentMap;
 import org.slf4j.Logger;
 
@@ -88,6 +90,8 @@
     protected MacAddress nodeMacAddr = null;
     protected LinkService linkService;
     protected FlowObjectiveService flowObjectiveService;
+    private DeviceConfiguration config;
+
     /**
      * local store for neighbor-device-ids and the set of ports on this device
      * that connect to the same neighbor.
@@ -106,6 +110,9 @@
     // distributed store for (device+subnet-ip-prefix) mapped to next-id
     protected EventuallyConsistentMap<VlanNextObjectiveStoreKey, Integer>
             vlanNextObjStore = null;
+    // distributed store for (device+mac+vlan+treatment) mapped to next-id
+    protected EventuallyConsistentMap<MacVlanNextObjectiveStoreKey, Integer>
+            macVlanNextObjStore = null;
     // distributed store for (device+port+treatment) mapped to next-id
     protected EventuallyConsistentMap<PortNextObjectiveStoreKey, Integer>
             portNextObjStore = null;
@@ -145,6 +152,7 @@
         this.dsNextObjStore = srManager.dsNextObjStore();
         this.vlanNextObjStore = srManager.vlanNextObjStore();
         this.portNextObjStore = srManager.portNextObjStore();
+        this.macVlanNextObjStore = srManager.macVlanNextObjStore();
         this.srManager = srManager;
         executorService.scheduleWithFixedDelay(new BucketCorrector(), 10,
                                                VERIFY_INTERVAL,
@@ -804,6 +812,52 @@
     }
 
     /**
+     * Returns the next objective of type simple associated with the mac/vlan on the
+     * device, given the treatment. Different treatments to the same mac/vlan result
+     * in different next objectives. If no such objective exists, this method
+     * creates one (if requested) and returns the id. Optionally metadata can be passed in for
+     * the creation of the objective. Typically this is used for L2 and L3 forwarding
+     * to compute nodes and containers/VMs on the compute nodes directly attached
+     * to the switch.
+     *
+     * @param macAddr the mac addr for the simple next objective
+     * @param vlanId the vlan for the simple next objective
+     * @param treatment the actions to apply on the packets (should include outport)
+     * @param meta optional metadata passed into the creation of the next objective
+     * @param createIfMissing true if a next object should be created if not found
+     * @return int if found or created, -1 if there are errors during the
+     *          creation of the next objective.
+     */
+    public int getMacVlanNextObjectiveId(MacAddress macAddr, VlanId vlanId, TrafficTreatment treatment,
+                                      TrafficSelector meta, boolean createIfMissing) {
+
+        Integer nextId = macVlanNextObjStore
+                .get(new MacVlanNextObjectiveStoreKey(deviceId, macAddr, vlanId));
+
+        if (nextId != null) {
+            return nextId;
+        }
+
+        log.debug("getMacVlanNextObjectiveId in device {}: Next objective id "
+                + "not found for host : {}/{} .. {}", deviceId, macAddr, vlanId,
+                (createIfMissing) ? "creating" : "aborting");
+
+        if (!createIfMissing) {
+            return -1;
+        }
+
+        /* create missing next objective */
+        nextId = createGroupFromMacVlan(macAddr, vlanId, treatment, meta);
+        if (nextId == null) {
+            log.warn("getMacVlanNextObjectiveId: unable to create next obj"
+                    + "for dev:{} host:{}/{}", deviceId, macAddr, vlanId);
+            return -1;
+        }
+        return nextId;
+    }
+
+
+    /**
      * Returns the next objective of type simple associated with the port on the
      * device, given the treatment. Different treatments to the same port result
      * in different next objectives. If no such objective exists, this method
@@ -1161,6 +1215,45 @@
     }
 
     /**
+     * Create simple next objective for an indirect host mac/vlan. The treatments can include
+     * all outgoing actions that need to happen on the packet.
+     *
+     * @param macAddr the mac address of the host
+     * @param vlanId the vlan of the host
+     * @param treatment the actions to apply on the packets (should include outport)
+     * @param meta optional data to pass to the driver
+     * @return next objective ID
+     */
+    public int createGroupFromMacVlan(MacAddress macAddr, VlanId vlanId, TrafficTreatment treatment,
+                                    TrafficSelector meta) {
+        int nextId = flowObjectiveService.allocateNextId();
+        MacVlanNextObjectiveStoreKey key = new MacVlanNextObjectiveStoreKey(deviceId, macAddr, vlanId);
+
+        NextObjective.Builder nextObjBuilder = DefaultNextObjective
+                .builder().withId(nextId)
+                .withType(NextObjective.Type.SIMPLE)
+                .addTreatment(treatment)
+                .fromApp(appId)
+                .withMeta(meta);
+
+        ObjectiveContext context = new DefaultObjectiveContext(
+            (objective) ->
+                log.debug("createGroupFromMacVlan installed "
+                        + "NextObj {} on {}", nextId, deviceId),
+            (objective, error) -> {
+                log.warn("createGroupFromMacVlan failed to install NextObj {} on {}: {}", nextId, deviceId, error);
+                srManager.invalidateNextObj(objective.id());
+            });
+        NextObjective nextObj = nextObjBuilder.add(context);
+        flowObjectiveService.next(deviceId, nextObj);
+        log.debug("createGroupFromMacVlan: Submited next objective {} in device {} "
+                + "for host {}/{}", nextId, deviceId, macAddr, vlanId);
+
+        macVlanNextObjStore.put(key, nextId);
+        return nextId;
+    }
+
+    /**
      * Create simple next objective for a single port. The treatments can include
      * all outgoing actions that need to happen on the packet.
      *
@@ -1241,6 +1334,7 @@
             portNextObjStore.remove(key);
         }
     }
+
     /**
      * Removes groups for the next objective ID given.
      *
@@ -1386,6 +1480,94 @@
     }
 
     /**
+     * Returns the next ID for the given host port from the store.
+     *
+     * @param hostMac mac of host for which Next ID is required.
+     * @param hostVlanId vlan of host for which Next ID is required.
+     * @param port port of device for which next ID is required.
+     * @return next objective ID or -1 if an error occurred during retrieval
+     */
+    public int getNextIdForHostPort(MacAddress hostMac, VlanId hostVlanId, PortNumber port) {
+
+       MacAddress deviceMac;
+        try {
+            deviceMac = deviceConfig.getDeviceMac(deviceId);
+        } catch (DeviceConfigNotFoundException e) {
+            log.warn(e.getMessage() + " in getNextIdForHostPort");
+            return -1;
+        }
+
+        TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
+        tbuilder.deferred()
+                .setEthDst(hostMac)
+                .setEthSrc(deviceMac)
+                .setVlanId(hostVlanId)
+                .setOutput(port);
+
+        TrafficSelector metadata =
+                 DefaultTrafficSelector.builder().matchVlanId(hostVlanId).build();
+
+        int nextId = getMacVlanNextObjectiveId(hostMac, hostVlanId,
+                tbuilder.build(), metadata, true);
+
+        log.debug("getNextId : hostMac {}, deviceMac {}, port {}, hostVlanId {} Treatment {} Meta {} nextId {} ",
+                                          hostMac, deviceMac, port, hostVlanId, tbuilder.build(), metadata, nextId);
+
+        return nextId;
+    }
+
+    /**
+     * Updates the next objective for the given nextId .
+     *
+     * @param hostMac mac of host for which Next obj is to be updated.
+     * @param hostVlanId vlan of host for which Next obj is to be updated.
+     * @param port port with which to update the Next Obj.
+     * @param nextId of Next Obj which needs to be updated.
+     */
+    public void updateL3UcastGroupBucket(MacAddress hostMac, VlanId hostVlanId, PortNumber port, int nextId) {
+
+       MacAddress deviceMac;
+        try {
+            deviceMac = deviceConfig.getDeviceMac(deviceId);
+        } catch (DeviceConfigNotFoundException e) {
+            log.warn(e.getMessage() + " in updateL3UcastGroupBucket");
+            return;
+        }
+
+        TrafficSelector metadata =
+                 DefaultTrafficSelector.builder().matchVlanId(hostVlanId).build();
+
+        TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
+        tbuilder.deferred()
+                .setEthDst(hostMac)
+                .setEthSrc(deviceMac)
+                .setVlanId(hostVlanId)
+                .setOutput(port);
+
+        log.debug(" update L3Ucast : deviceMac {}, port {}, host {}/{}, nextid {}, Treatment {} Meta {}",
+                                     deviceMac, port, hostMac, hostVlanId, nextId, tbuilder.build(), metadata);
+
+        NextObjective.Builder nextObjBuilder = DefaultNextObjective
+                .builder().withId(nextId)
+                .withType(NextObjective.Type.SIMPLE).fromApp(appId)
+                .addTreatment(tbuilder.build())
+                .withMeta(metadata);
+
+        ObjectiveContext context = new DefaultObjectiveContext(
+                (objective) -> log.debug(" NextId {} successfully updated host {} vlan {} with port {}",
+                                         nextId, hostMac, hostVlanId, port),
+                (objective, error) -> {
+                    log.warn(" NextId {} failed to update host {} vlan {} with port {}, error : {}",
+                                         nextId, hostMac, hostVlanId, port, error);
+                    srManager.invalidateNextObj(objective.id());
+                });
+
+        NextObjective nextObj = nextObjBuilder.modify(context);
+        flowObjectiveService.next(deviceId, nextObj);
+
+    }
+
+    /**
      * Adds a single port to the L2FG or removes it from the L2FG.
      *
      * @param vlanId the vlan id corresponding to this port
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/DestinationSetNextObjectiveStoreKey.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/DestinationSetNextObjectiveStoreKey.java
index 9b6e621..0baa769 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/DestinationSetNextObjectiveStoreKey.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/DestinationSetNextObjectiveStoreKey.java
@@ -21,6 +21,8 @@
 import org.onosproject.net.DeviceId;
 import org.onosproject.segmentrouting.grouphandler.DestinationSet;
 
+import static com.google.common.base.MoreObjects.toStringHelper;
+
 /**
  * Key of Destination set next objective store.
  */
@@ -84,6 +86,9 @@
 
     @Override
     public String toString() {
-        return "Device: " + deviceId + " " + ds;
+        return toStringHelper(getClass())
+                .add("deviceId", deviceId)
+                .add("ds", ds)
+                .toString();
     }
 }
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/MacVlanNextObjectiveStoreKey.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/MacVlanNextObjectiveStoreKey.java
new file mode 100644
index 0000000..4a1ab78
--- /dev/null
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/MacVlanNextObjectiveStoreKey.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.segmentrouting.storekey;
+
+import org.onlab.packet.VlanId;
+import org.onlab.packet.MacAddress;
+import org.onosproject.net.DeviceId;
+
+import java.util.Objects;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+
+/**
+ * Key of Device/Vlan/MacAddr to NextObjective store.
+ */
+public class MacVlanNextObjectiveStoreKey {
+    private final DeviceId deviceId;
+    private final MacAddress macAddr;
+    private final VlanId vlanId;
+
+    /**
+     * Constructs the key of the next objective store.
+     *
+     * @param deviceId device ID
+     * @param macAddr mac of host
+     * @param vlanId vlan of host
+     */
+    public MacVlanNextObjectiveStoreKey(DeviceId deviceId, MacAddress macAddr, VlanId vlanId) {
+        this.deviceId = deviceId;
+        this.macAddr = macAddr;
+        this.vlanId = vlanId;
+    }
+
+    /**
+     * Gets device id in this MacVlanNextObjectiveStoreKey.
+     *
+     * @return device id
+     */
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    /**
+     * Gets vlan information in this MacVlanNextObjectiveStoreKey.
+     *
+     * @return vlan information
+     */
+    public VlanId vlanId() {
+        return vlanId;
+    }
+
+    /**
+     * Gets mac information in this MacVlanNextObjectiveStoreKey.
+     *
+     * @return mac information
+     */
+    public MacAddress macAddr() {
+        return macAddr;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof MacVlanNextObjectiveStoreKey)) {
+            return false;
+        }
+        MacVlanNextObjectiveStoreKey that =
+                (MacVlanNextObjectiveStoreKey) o;
+        return (Objects.equals(this.deviceId, that.deviceId) &&
+                Objects.equals(this.vlanId, that.vlanId) &&
+                Objects.equals(this.macAddr, that.macAddr));
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(deviceId, vlanId, macAddr);
+    }
+
+    @Override
+    public String toString() {
+        return toStringHelper(getClass())
+                .add("deviceId", deviceId)
+                .add("vlanId", vlanId)
+                .add("macAddr", macAddr)
+                .toString();
+    }
+}
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/PortNextObjectiveStoreKey.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/PortNextObjectiveStoreKey.java
index 9aa41db..1269bce 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/PortNextObjectiveStoreKey.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/PortNextObjectiveStoreKey.java
@@ -22,6 +22,8 @@
 
 import java.util.Objects;
 
+import static com.google.common.base.MoreObjects.toStringHelper;
+
 /**
  * Key of Device/Port to NextObjective store.
  *
@@ -111,8 +113,11 @@
 
     @Override
     public String toString() {
-        return "ConnectPoint: " + deviceId + "/" + portNum +
-                " Treatment: " + treatment +
-                " Meta: " + meta;
+        return toStringHelper(getClass())
+                .add("deviceId", deviceId)
+                .add("portNum", portNum)
+                .add("treatment", treatment)
+                .add("meta", meta)
+                .toString();
     }
 }
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/VlanNextObjectiveStoreKey.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/VlanNextObjectiveStoreKey.java
index 5d9f2fd..55a95cd 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/VlanNextObjectiveStoreKey.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/storekey/VlanNextObjectiveStoreKey.java
@@ -21,6 +21,8 @@
 
 import java.util.Objects;
 
+import static com.google.common.base.MoreObjects.toStringHelper;
+
 /**
  * Key of VLAN to NextObjective store.
  */
@@ -79,6 +81,9 @@
 
     @Override
     public String toString() {
-        return "deviceId: " + deviceId + " vlanId: " + vlanId;
+        return toStringHelper(getClass())
+                .add("deviceId", deviceId)
+                .add("vlanId", vlanId)
+                .toString();
     }
 }