ONOS-3759 Group default provider

Change-Id: I318c8036a1836d13f57187bfd28464c740e09f08
diff --git a/core/api/src/main/java/org/onosproject/net/group/GroupProgrammable.java b/core/api/src/main/java/org/onosproject/net/group/GroupProgrammable.java
new file mode 100644
index 0000000..371817c
--- /dev/null
+++ b/core/api/src/main/java/org/onosproject/net/group/GroupProgrammable.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2016 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.net.group;
+
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.driver.HandlerBehaviour;
+
+/**
+ * Group programmable device behaviour.
+ */
+public interface GroupProgrammable extends HandlerBehaviour {
+
+    /**
+     * Performs the Group operations for the specified device.
+     *
+     * @param deviceId ID of the device
+     * @param groupOps operations to be performed
+     */
+    void performGroupOperation(DeviceId deviceId, GroupOperations groupOps);
+}
diff --git a/core/net/src/main/java/org/onosproject/net/group/impl/GroupDriverProvider.java b/core/net/src/main/java/org/onosproject/net/group/impl/GroupDriverProvider.java
new file mode 100644
index 0000000..75fbb89
--- /dev/null
+++ b/core/net/src/main/java/org/onosproject/net/group/impl/GroupDriverProvider.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2016 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.net.group.impl;
+
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.group.GroupOperations;
+import org.onosproject.net.group.GroupProgrammable;
+import org.onosproject.net.group.GroupProvider;
+import org.onosproject.net.provider.AbstractProvider;
+import org.onosproject.net.provider.ProviderId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Driver-based Group rule provider.
+ */
+public class GroupDriverProvider extends AbstractProvider implements GroupProvider {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
+    // To be extracted for reuse as we deal with other.
+    private static final String SCHEME = "default";
+    private static final String PROVIDER_NAME = "org.onosproject.provider";
+    protected DeviceService deviceService;
+
+    public GroupDriverProvider() {
+        super(new ProviderId(SCHEME, PROVIDER_NAME));
+    }
+
+    /**
+     * Initializes the provider with the necessary device service.
+     *
+     * @param deviceService device service
+     */
+    void init(DeviceService deviceService) {
+        this.deviceService = deviceService;
+    }
+
+    @Override
+    public void performGroupOperation(DeviceId deviceId, GroupOperations groupOps) {
+        GroupProgrammable programmable = getGroupProgrammable(deviceId);
+        if (programmable != null) {
+            programmable.performGroupOperation(deviceId, groupOps);
+        }
+    }
+
+    private GroupProgrammable getGroupProgrammable(DeviceId deviceId) {
+        GroupProgrammable programmable = deviceService.getDevice(deviceId).as(GroupProgrammable.class);
+        if (programmable == null) {
+            log.warn("Device {} is not group programmable");
+        }
+        return programmable;
+    }
+}
diff --git a/core/net/src/main/java/org/onosproject/net/group/impl/GroupManager.java b/core/net/src/main/java/org/onosproject/net/group/impl/GroupManager.java
index d6158b5..e9d76fb 100644
--- a/core/net/src/main/java/org/onosproject/net/group/impl/GroupManager.java
+++ b/core/net/src/main/java/org/onosproject/net/group/impl/GroupManager.java
@@ -87,6 +87,7 @@
     @Property(name = "purgeOnDisconnection", boolValue = false,
             label = "Purge entries associated with a device when the device goes offline")
     private boolean purgeOnDisconnection = false;
+    private final  GroupDriverProvider defaultProvider = new GroupDriverProvider();
 
     @Activate
     public void activate(ComponentContext context) {
@@ -111,6 +112,12 @@
         if (context != null) {
             readComponentConfiguration(context);
         }
+        defaultProvider.init(deviceService);
+    }
+
+    @Override
+    protected GroupProvider defaultProvider() {
+        return defaultProvider;
     }
 
     /**
diff --git a/core/net/src/test/java/org/onosproject/net/group/impl/GroupManagerTest.java b/core/net/src/test/java/org/onosproject/net/group/impl/GroupManagerTest.java
index 27aaaf9..a171063 100644
--- a/core/net/src/test/java/org/onosproject/net/group/impl/GroupManagerTest.java
+++ b/core/net/src/test/java/org/onosproject/net/group/impl/GroupManagerTest.java
@@ -15,32 +15,30 @@
  */
 package org.onosproject.net.group.impl;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.onosproject.net.NetTestTools.injectEventDispatcher;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.MplsLabel;
 import org.onosproject.cfg.ComponentConfigAdapter;
+import org.onosproject.common.event.impl.TestEventDispatcher;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.DefaultApplicationId;
 import org.onosproject.core.DefaultGroupId;
 import org.onosproject.core.GroupId;
-import org.onosproject.common.event.impl.TestEventDispatcher;
+import org.onosproject.net.AnnotationKeys;
+import org.onosproject.net.DefaultAnnotations;
+import org.onosproject.net.DefaultDevice;
+import org.onosproject.net.Device;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.PortNumber;
-import org.onosproject.net.device.impl.DeviceManager;
+import org.onosproject.net.device.DeviceServiceAdapter;
+import org.onosproject.net.driver.AbstractHandlerBehaviour;
+import org.onosproject.net.driver.DefaultDriver;
+import org.onosproject.net.driver.impl.DriverManager;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.group.DefaultGroup;
@@ -56,6 +54,7 @@
 import org.onosproject.net.group.GroupListener;
 import org.onosproject.net.group.GroupOperation;
 import org.onosproject.net.group.GroupOperations;
+import org.onosproject.net.group.GroupProgrammable;
 import org.onosproject.net.group.GroupProvider;
 import org.onosproject.net.group.GroupProviderRegistry;
 import org.onosproject.net.group.GroupProviderService;
@@ -65,7 +64,13 @@
 import org.onosproject.net.provider.ProviderId;
 import org.onosproject.store.trivial.SimpleGroupStore;
 
-import com.google.common.collect.Iterables;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.*;
+import static org.onosproject.net.NetTestTools.injectEventDispatcher;
 
 /**
  * Test codifying the group service & group provider service contracts.
@@ -74,6 +79,14 @@
 
     private static final ProviderId PID = new ProviderId("of", "groupfoo");
     private static final DeviceId DID = DeviceId.deviceId("of:001");
+    private static final ProviderId FOO_PID = new ProviderId("foo", "foo");
+    private static final DeviceId FOO_DID = DeviceId.deviceId("foo:002");
+
+    private static final DefaultAnnotations ANNOTATIONS =
+            DefaultAnnotations.builder().set(AnnotationKeys.DRIVER, "foo").build();
+
+    private static final Device FOO_DEV =
+            new DefaultDevice(FOO_PID, FOO_DID, Device.Type.SWITCH, "", "", "", "", null, ANNOTATIONS);
 
     private GroupManager mgr;
     private GroupService groupService;
@@ -84,12 +97,14 @@
     private GroupProvider provider;
     private GroupProviderService providerService;
     private ApplicationId appId;
+    private TestDriverManager driverService;
 
     @Before
     public void setUp() {
         mgr = new GroupManager();
         groupService = mgr;
-        mgr.deviceService = new DeviceManager();
+        //mgr.deviceService = new DeviceManager();
+        mgr.deviceService = new TestDeviceService();
         mgr.cfgService = new ComponentConfigAdapter();
         mgr.store = new SimpleGroupStore();
         injectEventDispatcher(mgr, new TestEventDispatcher());
@@ -98,6 +113,12 @@
         mgr.activate(null);
         mgr.addListener(listener);
 
+        driverService = new TestDriverManager();
+        driverService.addDriver(new DefaultDriver("foo", ImmutableList.of(), "", "", "",
+                                                  ImmutableMap.of(GroupProgrammable.class,
+                                                                  TestGroupProgrammable.class),
+                                                  ImmutableMap.of()));
+
         internalProvider = new TestGroupProvider(PID);
         provider = internalProvider;
         providerService = providerRegistry.register(provider);
@@ -117,55 +138,170 @@
     }
 
     /**
-     * Tests group service north bound and south bound interfaces.
-     * The following operations are tested:
-     * a)Tests group creation before the device group AUDIT completes
-     * b)Tests initial device group AUDIT process
-     * c)Tests deletion process of any extraneous groups
-     * d)Tests execution of any pending group creation requests
-     * after the device group AUDIT completes
-     * e)Tests re-apply process of any missing groups
-     * f)Tests event notifications after receiving confirmation for
-     * any operations from data plane
-     * g)Tests group bucket modifications (additions and deletions)
-     * h)Tests group deletion
+     * Tests group creation before the device group AUDIT completes.
      */
     @Test
-    public void testGroupService() {
+    public void testGroupServiceBasics() {
         // Test Group creation before AUDIT process
-        testGroupCreationBeforeAudit();
+        testGroupCreationBeforeAudit(DID);
+    }
 
+    /**
+     * Tests initial device group AUDIT process.
+     */
+    @Test
+    public void testGroupServiceInitialAudit() {
+        // Test Group creation before AUDIT process
+        testGroupCreationBeforeAudit(DID);
         // Test initial group audit process
-        testInitialAuditWithPendingGroupRequests();
+        testInitialAuditWithPendingGroupRequests(DID);
+    }
+
+    /**
+     * Tests deletion process of any extraneous groups.
+     */
+    @Test
+    public void testGroupServiceAuditExtraneous() {
+        // Test Group creation before AUDIT process
+        testGroupCreationBeforeAudit(DID);
 
         // Test audit with extraneous and missing groups
-        testAuditWithExtraneousMissingGroups();
+        testAuditWithExtraneousMissingGroups(DID);
+    }
+
+    /**
+     * Tests re-apply process of any missing groups tests execution of
+     * any pending group creation request after the device group AUDIT completes
+     * and tests event notifications after receiving confirmation for any
+     * operations from data plane.
+     */
+    @Test
+    public void testGroupServiceAuditConfirmed() {
+        // Test Group creation before AUDIT process
+        testGroupCreationBeforeAudit(DID);
+
+        // Test audit with extraneous and missing groups
+        testAuditWithExtraneousMissingGroups(DID);
 
         // Test audit with confirmed groups
-        testAuditWithConfirmedGroups();
+        testAuditWithConfirmedGroups(DID);
+    }
 
+    /**
+     * Tests group bucket modifications (additions and deletions) and
+     * Tests group deletion.
+     */
+    @Test
+    public void testGroupServiceBuckets() {
+        // Test Group creation before AUDIT process
+        testGroupCreationBeforeAudit(DID);
+        programmableTestCleanUp();
+
+        testAuditWithExtraneousMissingGroups(DID);
         // Test group add bucket operations
-        testAddBuckets();
+        testAddBuckets(DID);
 
         // Test group remove bucket operations
-        testRemoveBuckets();
+        testRemoveBuckets(DID);
 
         // Test group remove operations
-        testRemoveGroup();
+        testRemoveGroup(DID);
+    }
+
+    /**
+     * Tests group creation before the device group AUDIT completes with fallback
+     * provider.
+     */
+    @Test
+    public void testGroupServiceFallbackBasics() {
+        // Test Group creation before AUDIT process
+        testGroupCreationBeforeAudit(FOO_DID);
+        programmableTestCleanUp();
+
+    }
+
+    /**
+     * Tests initial device group AUDIT process with fallback provider.
+     */
+    @Test
+    public void testGroupServiceFallbackInitialAudit() {
+        // Test Group creation before AUDIT process
+        testGroupCreationBeforeAudit(FOO_DID);
+        programmableTestCleanUp();
+        // Test initial group audit process
+        testInitialAuditWithPendingGroupRequests(FOO_DID);
+    }
+
+    /**
+     * Tests deletion process of any extraneous groups with fallback provider.
+     */
+    @Test
+    public void testGroupServiceFallbackAuditExtraneous() {
+        // Test Group creation before AUDIT process
+        testGroupCreationBeforeAudit(FOO_DID);
+        programmableTestCleanUp();
+
+        // Test audit with extraneous and missing groups
+        testAuditWithExtraneousMissingGroups(FOO_DID);
+    }
+
+    /**
+     * Tests re-apply process of any missing groups tests execution of
+     * any pending group creation request after the device group AUDIT completes
+     * and tests event notifications after receiving confirmation for any
+     * operations from data plane with fallback provider.
+     */
+    @Test
+    public void testGroupServiceFallbackAuditConfirmed() {
+        // Test Group creation before AUDIT process
+        testGroupCreationBeforeAudit(FOO_DID);
+        programmableTestCleanUp();
+
+        // Test audit with extraneous and missing groups
+        testAuditWithExtraneousMissingGroups(FOO_DID);
+
+        // Test audit with confirmed groups
+        testAuditWithConfirmedGroups(FOO_DID);
+    }
+
+    /**
+     * Tests group bucket modifications (additions and deletions) and
+     * Tests group deletion with fallback provider.
+     */
+    @Test
+    public void testGroupServiceFallbackBuckets() {
+        // Test Group creation before AUDIT process
+        testGroupCreationBeforeAudit(FOO_DID);
+        programmableTestCleanUp();
+
+        testAuditWithExtraneousMissingGroups(FOO_DID);
+        // Test group add bucket operations
+        testAddBuckets(FOO_DID);
+
+        // Test group remove bucket operations
+        testRemoveBuckets(FOO_DID);
+
+        // Test group remove operations
+        testRemoveGroup(FOO_DID);
+    }
+
+    private void programmableTestCleanUp() {
+        groupOperations.clear();
+        lastDeviceIdProgrammable = null;
     }
 
     // Test Group creation before AUDIT process
-    private void testGroupCreationBeforeAudit() {
+    private void testGroupCreationBeforeAudit(DeviceId deviceId) {
         PortNumber[] ports1 = {PortNumber.portNumber(31),
-                               PortNumber.portNumber(32)};
+                PortNumber.portNumber(32)};
         PortNumber[] ports2 = {PortNumber.portNumber(41),
-                               PortNumber.portNumber(42)};
+                PortNumber.portNumber(42)};
         GroupKey key = new DefaultGroupKey("group1BeforeAudit".getBytes());
         List<GroupBucket> buckets = new ArrayList<>();
         List<PortNumber> outPorts = new ArrayList<>();
         outPorts.addAll(Arrays.asList(ports1));
         outPorts.addAll(Arrays.asList(ports2));
-        for (PortNumber portNumber: outPorts) {
+        for (PortNumber portNumber : outPorts) {
             TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
             tBuilder.setOutput(portNumber)
                     .setEthDst(MacAddress.valueOf("00:00:00:00:00:02"))
@@ -173,75 +309,78 @@
                     .pushMpls()
                     .setMpls(MplsLabel.mplsLabel(106));
             buckets.add(DefaultGroupBucket.createSelectGroupBucket(
-                                                        tBuilder.build()));
+                    tBuilder.build()));
         }
         GroupBuckets groupBuckets = new GroupBuckets(buckets);
-        GroupDescription newGroupDesc = new DefaultGroupDescription(DID,
+        GroupDescription newGroupDesc = new DefaultGroupDescription(deviceId,
                                                                     Group.Type.SELECT,
                                                                     groupBuckets,
                                                                     key,
                                                                     null,
                                                                     appId);
         groupService.addGroup(newGroupDesc);
-        internalProvider.validate(DID, null);
-        assertEquals(null, groupService.getGroup(DID, key));
-        assertEquals(0, Iterables.size(groupService.getGroups(DID, appId)));
+        assertEquals(null, groupService.getGroup(deviceId, key));
+        assertEquals(0, Iterables.size(groupService.getGroups(deviceId, appId)));
     }
 
     // Test initial AUDIT process with pending group requests
-    private void testInitialAuditWithPendingGroupRequests() {
+    private void testInitialAuditWithPendingGroupRequests(DeviceId deviceId) {
         PortNumber[] ports1 = {PortNumber.portNumber(31),
-                               PortNumber.portNumber(32)};
+                PortNumber.portNumber(32)};
         PortNumber[] ports2 = {PortNumber.portNumber(41),
-                               PortNumber.portNumber(42)};
+                PortNumber.portNumber(42)};
         GroupId gId1 = new DefaultGroupId(1);
         Group group1 = createSouthboundGroupEntry(gId1,
                                                   Arrays.asList(ports1),
-                                                  0);
+                                                  0, deviceId);
         GroupId gId2 = new DefaultGroupId(2);
         // Non zero reference count will make the group manager to queue
         // the extraneous groups until reference count is zero.
         Group group2 = createSouthboundGroupEntry(gId2,
                                                   Arrays.asList(ports2),
-                                                  2);
+                                                  2, deviceId);
         List<Group> groupEntries = Arrays.asList(group1, group2);
-        providerService.pushGroupMetrics(DID, groupEntries);
+        providerService.pushGroupMetrics(deviceId, groupEntries);
         // First group metrics would trigger the device audit completion
         // post which all pending group requests are also executed.
         GroupKey key = new DefaultGroupKey("group1BeforeAudit".getBytes());
-        Group createdGroup = groupService.getGroup(DID, key);
+        Group createdGroup = groupService.getGroup(deviceId, key);
         int createdGroupId = createdGroup.id().id();
         assertNotEquals(gId1.id(), createdGroupId);
         assertNotEquals(gId2.id(), createdGroupId);
 
         List<GroupOperation> expectedGroupOps = Arrays.asList(
-                            GroupOperation.createDeleteGroupOperation(gId1,
+                GroupOperation.createDeleteGroupOperation(gId1,
                                                           Group.Type.SELECT),
-                            GroupOperation.createAddGroupOperation(
-                                           createdGroup.id(),
-                                           Group.Type.SELECT,
-                                           createdGroup.buckets()));
-        internalProvider.validate(DID, expectedGroupOps);
+                GroupOperation.createAddGroupOperation(
+                        createdGroup.id(),
+                        Group.Type.SELECT,
+                        createdGroup.buckets()));
+        if (deviceId.equals(DID)) {
+            internalProvider.validate(deviceId, expectedGroupOps);
+        } else {
+            this.validate(deviceId, expectedGroupOps);
+        }
     }
 
     // Test AUDIT process with extraneous groups and missing groups
-    private void testAuditWithExtraneousMissingGroups() {
+    private void testAuditWithExtraneousMissingGroups(DeviceId deviceId) {
         PortNumber[] ports1 = {PortNumber.portNumber(31),
-                               PortNumber.portNumber(32)};
+                PortNumber.portNumber(32)};
         PortNumber[] ports2 = {PortNumber.portNumber(41),
-                               PortNumber.portNumber(42)};
+                PortNumber.portNumber(42)};
         GroupId gId1 = new DefaultGroupId(1);
         Group group1 = createSouthboundGroupEntry(gId1,
-                                            Arrays.asList(ports1),
-                                            0);
+                                                  Arrays.asList(ports1),
+                                                  0, deviceId);
         GroupId gId2 = new DefaultGroupId(2);
         Group group2 = createSouthboundGroupEntry(gId2,
-                                            Arrays.asList(ports2),
-                                            0);
+                                                  Arrays.asList(ports2),
+                                                  0, deviceId);
         List<Group> groupEntries = Arrays.asList(group1, group2);
-        providerService.pushGroupMetrics(DID, groupEntries);
+        providerService.pushGroupMetrics(deviceId, groupEntries);
         GroupKey key = new DefaultGroupKey("group1BeforeAudit".getBytes());
-        Group createdGroup = groupService.getGroup(DID, key);
+        Group createdGroup = groupService.getGroup(deviceId, key);
         List<GroupOperation> expectedGroupOps = Arrays.asList(
                 GroupOperation.createDeleteGroupOperation(gId1,
                                                           Group.Type.SELECT),
@@ -250,39 +389,43 @@
                 GroupOperation.createAddGroupOperation(createdGroup.id(),
                                                        Group.Type.SELECT,
                                                        createdGroup.buckets()));
-        internalProvider.validate(DID, expectedGroupOps);
+        if (deviceId.equals(DID)) {
+            internalProvider.validate(deviceId, expectedGroupOps);
+        } else {
+            this.validate(deviceId, expectedGroupOps);
+        }
     }
 
     // Test AUDIT with confirmed groups
-    private void testAuditWithConfirmedGroups() {
+    private void testAuditWithConfirmedGroups(DeviceId deviceId) {
         GroupKey key = new DefaultGroupKey("group1BeforeAudit".getBytes());
-        Group createdGroup = groupService.getGroup(DID, key);
+        Group createdGroup = groupService.getGroup(deviceId, key);
         createdGroup = new DefaultGroup(createdGroup.id(),
-                                        DID,
+                                        deviceId,
                                         Group.Type.SELECT,
                                         createdGroup.buckets());
         List<Group> groupEntries = Collections.singletonList(createdGroup);
-        providerService.pushGroupMetrics(DID, groupEntries);
+        providerService.pushGroupMetrics(deviceId, groupEntries);
         internalListener.validateEvent(Collections.singletonList(GroupEvent.Type.GROUP_ADDED));
     }
 
     // Test group add bucket operations
-    private void testAddBuckets() {
+    private void testAddBuckets(DeviceId deviceId) {
         GroupKey addKey = new DefaultGroupKey("group1AddBuckets".getBytes());
 
         GroupKey prevKey = new DefaultGroupKey("group1BeforeAudit".getBytes());
-        Group createdGroup = groupService.getGroup(DID, prevKey);
+        Group createdGroup = groupService.getGroup(deviceId, prevKey);
         List<GroupBucket> buckets = new ArrayList<>();
         buckets.addAll(createdGroup.buckets().buckets());
 
         PortNumber[] addPorts = {PortNumber.portNumber(51),
-                                 PortNumber.portNumber(52)};
+                PortNumber.portNumber(52)};
         List<PortNumber> outPorts;
         outPorts = new ArrayList<>();
         outPorts.addAll(Arrays.asList(addPorts));
         List<GroupBucket> addBuckets;
         addBuckets = new ArrayList<>();
-        for (PortNumber portNumber: outPorts) {
+        for (PortNumber portNumber : outPorts) {
             TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
             tBuilder.setOutput(portNumber)
                     .setEthDst(MacAddress.valueOf("00:00:00:00:00:02"))
@@ -290,12 +433,12 @@
                     .pushMpls()
                     .setMpls(MplsLabel.mplsLabel(106));
             addBuckets.add(DefaultGroupBucket.createSelectGroupBucket(
-                                                        tBuilder.build()));
+                    tBuilder.build()));
             buckets.add(DefaultGroupBucket.createSelectGroupBucket(
-                                                        tBuilder.build()));
+                    tBuilder.build()));
         }
         GroupBuckets groupAddBuckets = new GroupBuckets(addBuckets);
-        groupService.addBucketsToGroup(DID,
+        groupService.addBucketsToGroup(deviceId,
                                        prevKey,
                                        groupAddBuckets,
                                        addKey,
@@ -303,30 +446,34 @@
         GroupBuckets updatedBuckets = new GroupBuckets(buckets);
         List<GroupOperation> expectedGroupOps = Collections.singletonList(
                 GroupOperation.createModifyGroupOperation(createdGroup.id(),
-                        Group.Type.SELECT,
-                        updatedBuckets));
-        internalProvider.validate(DID, expectedGroupOps);
-        Group existingGroup = groupService.getGroup(DID, addKey);
+                                                          Group.Type.SELECT,
+                                                          updatedBuckets));
+        if (deviceId.equals(DID)) {
+            internalProvider.validate(deviceId, expectedGroupOps);
+        } else {
+            this.validate(deviceId, expectedGroupOps);
+        }
+        Group existingGroup = groupService.getGroup(deviceId, addKey);
         List<Group> groupEntries = Collections.singletonList(existingGroup);
-        providerService.pushGroupMetrics(DID, groupEntries);
+        providerService.pushGroupMetrics(deviceId, groupEntries);
         internalListener.validateEvent(Collections.singletonList(GroupEvent.Type.GROUP_UPDATED));
     }
 
     // Test group remove bucket operations
-    private void testRemoveBuckets() {
+    private void testRemoveBuckets(DeviceId deviceId) {
         GroupKey removeKey = new DefaultGroupKey("group1RemoveBuckets".getBytes());
 
         GroupKey prevKey = new DefaultGroupKey("group1AddBuckets".getBytes());
-        Group createdGroup = groupService.getGroup(DID, prevKey);
+        Group createdGroup = groupService.getGroup(deviceId, prevKey);
         List<GroupBucket> buckets = new ArrayList<>();
         buckets.addAll(createdGroup.buckets().buckets());
 
         PortNumber[] removePorts = {PortNumber.portNumber(31),
-                                 PortNumber.portNumber(32)};
+                PortNumber.portNumber(32)};
         List<PortNumber> outPorts = new ArrayList<>();
         outPorts.addAll(Arrays.asList(removePorts));
         List<GroupBucket> removeBuckets = new ArrayList<>();
-        for (PortNumber portNumber: outPorts) {
+        for (PortNumber portNumber : outPorts) {
             TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
             tBuilder.setOutput(portNumber)
                     .setEthDst(MacAddress.valueOf("00:00:00:00:00:02"))
@@ -334,12 +481,12 @@
                     .pushMpls()
                     .setMpls(MplsLabel.mplsLabel(106));
             removeBuckets.add(DefaultGroupBucket.createSelectGroupBucket(
-                                                        tBuilder.build()));
+                    tBuilder.build()));
             buckets.remove(DefaultGroupBucket.createSelectGroupBucket(
-                                                        tBuilder.build()));
+                    tBuilder.build()));
         }
         GroupBuckets groupRemoveBuckets = new GroupBuckets(removeBuckets);
-        groupService.removeBucketsFromGroup(DID,
+        groupService.removeBucketsFromGroup(deviceId,
                                             prevKey,
                                             groupRemoveBuckets,
                                             removeKey,
@@ -347,26 +494,34 @@
         GroupBuckets updatedBuckets = new GroupBuckets(buckets);
         List<GroupOperation> expectedGroupOps = Collections.singletonList(
                 GroupOperation.createModifyGroupOperation(createdGroup.id(),
-                        Group.Type.SELECT,
-                        updatedBuckets));
-        internalProvider.validate(DID, expectedGroupOps);
-        Group existingGroup = groupService.getGroup(DID, removeKey);
+                                                          Group.Type.SELECT,
+                                                          updatedBuckets));
+        if (deviceId.equals(DID)) {
+            internalProvider.validate(deviceId, expectedGroupOps);
+        } else {
+            this.validate(deviceId, expectedGroupOps);
+        }
+        Group existingGroup = groupService.getGroup(deviceId, removeKey);
         List<Group> groupEntries = Collections.singletonList(existingGroup);
-        providerService.pushGroupMetrics(DID, groupEntries);
+        providerService.pushGroupMetrics(deviceId, groupEntries);
         internalListener.validateEvent(Collections.singletonList(GroupEvent.Type.GROUP_UPDATED));
     }
 
     // Test group remove operations
-    private void testRemoveGroup() {
+    private void testRemoveGroup(DeviceId deviceId) {
         GroupKey currKey = new DefaultGroupKey("group1RemoveBuckets".getBytes());
-        Group existingGroup = groupService.getGroup(DID, currKey);
-        groupService.removeGroup(DID, currKey, appId);
+        Group existingGroup = groupService.getGroup(deviceId, currKey);
+        groupService.removeGroup(deviceId, currKey, appId);
         List<GroupOperation> expectedGroupOps = Collections.singletonList(
                 GroupOperation.createDeleteGroupOperation(existingGroup.id(),
-                        Group.Type.SELECT));
-        internalProvider.validate(DID, expectedGroupOps);
+                                                          Group.Type.SELECT));
+        if (deviceId.equals(DID)) {
+            internalProvider.validate(deviceId, expectedGroupOps);
+        } else {
+            this.validate(deviceId, expectedGroupOps);
+        }
         List<Group> groupEntries = Collections.emptyList();
-        providerService.pushGroupMetrics(DID, groupEntries);
+        providerService.pushGroupMetrics(deviceId, groupEntries);
         internalListener.validateEvent(Collections.singletonList(GroupEvent.Type.GROUP_REMOVED));
     }
 
@@ -378,6 +533,22 @@
      */
     @Test
     public void testGroupOperationFailure() {
+        groupOperationFaliure(DID);
+    }
+
+    /**
+     * Test GroupOperationFailure function in Group Manager
+     * with fallback provider.
+     * a)GroupAddFailure
+     * b)GroupUpdateFailure
+     * c)GroupRemoteFailure
+     */
+    @Test
+    public void testGroupOperationFailureFallBack() {
+        groupOperationFaliure(FOO_DID);
+    }
+
+    private void groupOperationFaliure(DeviceId deviceId) {
         PortNumber[] ports1 = {PortNumber.portNumber(31),
                 PortNumber.portNumber(32)};
         PortNumber[] ports2 = {PortNumber.portNumber(41),
@@ -388,7 +559,7 @@
         List<PortNumber> outPorts = new ArrayList<>();
         outPorts.addAll(Arrays.asList(ports1));
         outPorts.addAll(Arrays.asList(ports2));
-        for (PortNumber portNumber: outPorts) {
+        for (PortNumber portNumber : outPorts) {
             TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
             tBuilder.setOutput(portNumber)
                     .setEthDst(MacAddress.valueOf("00:00:00:00:00:02"))
@@ -399,70 +570,69 @@
                     tBuilder.build()));
         }
         GroupBuckets groupBuckets = new GroupBuckets(buckets);
-        GroupDescription newGroupDesc = new DefaultGroupDescription(DID,
-                Group.Type.SELECT,
-                groupBuckets,
-                key,
-                null,
-                appId);
+        GroupDescription newGroupDesc = new DefaultGroupDescription(deviceId,
+                                                                    Group.Type.SELECT,
+                                                                    groupBuckets,
+                                                                    key,
+                                                                    null,
+                                                                    appId);
         groupService.addGroup(newGroupDesc);
 
         // Test initial group audit process
         GroupId gId1 = new DefaultGroupId(1);
         Group group1 = createSouthboundGroupEntry(gId1,
-                Arrays.asList(ports1),
-                0);
+                                                  Arrays.asList(ports1),
+                                                  0, deviceId);
         GroupId gId2 = new DefaultGroupId(2);
         // Non zero reference count will make the group manager to queue
         // the extraneous groups until reference count is zero.
         Group group2 = createSouthboundGroupEntry(gId2,
-                Arrays.asList(ports2),
-                2);
+                                                  Arrays.asList(ports2),
+                                                  2, deviceId);
         List<Group> groupEntries = Arrays.asList(group1, group2);
-        providerService.pushGroupMetrics(DID, groupEntries);
-        Group createdGroup = groupService.getGroup(DID, key);
+        providerService.pushGroupMetrics(deviceId, groupEntries);
+        Group createdGroup = groupService.getGroup(deviceId, key);
 
         // Group Add failure test
         GroupOperation groupAddOp = GroupOperation.
                 createAddGroupOperation(createdGroup.id(),
-                        createdGroup.type(),
-                        createdGroup.buckets());
-        providerService.groupOperationFailed(DID, groupAddOp);
+                                        createdGroup.type(),
+                                        createdGroup.buckets());
+        providerService.groupOperationFailed(deviceId, groupAddOp);
         internalListener.validateEvent(Collections.singletonList(GroupEvent.Type.GROUP_ADD_FAILED));
 
         // Group Mod failure test
         groupService.addGroup(newGroupDesc);
-        createdGroup = groupService.getGroup(DID, key);
+        createdGroup = groupService.getGroup(deviceId, key);
         assertNotNull(createdGroup);
 
         GroupOperation groupModOp = GroupOperation.
                 createModifyGroupOperation(createdGroup.id(),
-                        createdGroup.type(),
-                        createdGroup.buckets());
-        providerService.groupOperationFailed(DID, groupModOp);
+                                           createdGroup.type(),
+                                           createdGroup.buckets());
+        providerService.groupOperationFailed(deviceId, groupModOp);
         internalListener.validateEvent(Collections.singletonList(GroupEvent.Type.GROUP_UPDATE_FAILED));
 
         // Group Delete failure test
         groupService.addGroup(newGroupDesc);
-        createdGroup = groupService.getGroup(DID, key);
+        createdGroup = groupService.getGroup(deviceId, key);
         assertNotNull(createdGroup);
 
         GroupOperation groupDelOp = GroupOperation.
                 createDeleteGroupOperation(createdGroup.id(),
-                        createdGroup.type());
-        providerService.groupOperationFailed(DID, groupDelOp);
+                                           createdGroup.type());
+        providerService.groupOperationFailed(deviceId, groupDelOp);
         internalListener.validateEvent(Collections.singletonList(GroupEvent.Type.GROUP_REMOVE_FAILED));
-
     }
 
     private Group createSouthboundGroupEntry(GroupId gId,
                                              List<PortNumber> ports,
-                                             long referenceCount) {
+                                             long referenceCount, DeviceId deviceId) {
         List<PortNumber> outPorts = new ArrayList<>();
         outPorts.addAll(ports);
 
         List<GroupBucket> buckets = new ArrayList<>();
-        for (PortNumber portNumber: outPorts) {
+        for (PortNumber portNumber : outPorts) {
             TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
             tBuilder.setOutput(portNumber)
                     .setEthDst(MacAddress.valueOf("00:00:00:00:00:02"))
@@ -470,11 +640,11 @@
                     .pushMpls()
                     .setMpls(MplsLabel.mplsLabel(106));
             buckets.add(DefaultGroupBucket.createSelectGroupBucket(
-                                                        tBuilder.build()));
+                    tBuilder.build()));
         }
         GroupBuckets groupBuckets = new GroupBuckets(buckets);
         StoredGroupEntry group = new DefaultGroup(
-                            gId, DID, Group.Type.SELECT, groupBuckets);
+                gId, deviceId, Group.Type.SELECT, groupBuckets);
         group.setReferenceCount(referenceCount);
         return group;
     }
@@ -501,7 +671,7 @@
     }
 
     private class TestGroupProvider
-                extends AbstractProvider implements GroupProvider {
+            extends AbstractProvider implements GroupProvider {
         DeviceId lastDeviceId;
         List<GroupOperation> groupOperations = new ArrayList<>();
 
@@ -533,6 +703,60 @@
 
     }
 
+    private static class TestDeviceService extends DeviceServiceAdapter {
+        @Override
+        public int getDeviceCount() {
+            return 1;
+        }
+
+        @Override
+        public Iterable<Device> getDevices() {
+            return ImmutableList.of(FOO_DEV);
+        }
+
+        @Override
+        public Iterable<Device> getAvailableDevices() {
+            return getDevices();
+        }
+
+        @Override
+        public Device getDevice(DeviceId deviceId) {
+            return FOO_DEV;
+        }
+    }
+
+    private class TestDriverManager extends DriverManager {
+        TestDriverManager() {
+            this.deviceService = mgr.deviceService;
+            activate();
+        }
+    }
+
+    private static DeviceId lastDeviceIdProgrammable;
+    private static List<GroupOperation> groupOperations = new ArrayList<>();
+
+    public static class TestGroupProgrammable extends AbstractHandlerBehaviour implements GroupProgrammable {
+        @Override
+        public void performGroupOperation(DeviceId deviceId, GroupOperations groupOps) {
+            lastDeviceIdProgrammable = deviceId;
+            groupOperations.addAll(groupOps.operations());
+        }
+    }
+
+    public void validate(DeviceId expectedDeviceId,
+                         List<GroupOperation> expectedGroupOps) {
+        if (expectedGroupOps == null) {
+            assertTrue("events generated", groupOperations.isEmpty());
+            return;
+        }
+
+        assertEquals(lastDeviceIdProgrammable, expectedDeviceId);
+        assertTrue((this.groupOperations.containsAll(expectedGroupOps) &&
+                expectedGroupOps.containsAll(groupOperations)));
+
+        groupOperations.clear();
+        lastDeviceIdProgrammable = null;
+    }
 }