[ONOS-5729] VirtualFlowRuleStore implementation

Changes
1. VirtualStore and AbstractVirtualStore are defined
2. SimpleFlowRuleStore is added
3. FlowRuleStoreDeligate is implemented

Change-Id: Ic1f3d25f8a52d8c9bb6bc4de67abc0d642f47c9c
diff --git a/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowRuleManager.java b/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowRuleManager.java
index 0ac256e..31c1415 100644
--- a/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowRuleManager.java
+++ b/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowRuleManager.java
@@ -49,7 +49,9 @@
 import org.onosproject.net.flow.FlowRuleOperation;
 import org.onosproject.net.flow.FlowRuleOperations;
 import org.onosproject.net.flow.FlowRuleService;
+import org.onosproject.net.flow.FlowRuleStoreDelegate;
 import org.onosproject.net.flow.TableStatisticsEntry;
+import org.onosproject.net.provider.ProviderId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -64,6 +66,8 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.net.flow.FlowRuleEvent.Type.RULE_ADD_REQUESTED;
+import static org.onosproject.net.flow.FlowRuleEvent.Type.RULE_REMOVE_REQUESTED;
 
 /**
  * Flow rule service implementation built on the virtual network service.
@@ -98,6 +102,8 @@
     private VirtualProviderRegistryService providerRegistryService = null;
     private InternalFlowRuleProviderService innerProviderService = null;
 
+    private final FlowRuleStoreDelegate storeDelegate;
+
     /**
      * Creates a new VirtualNetworkFlowRuleService object.
      *
@@ -109,6 +115,7 @@
         super(virtualNetworkManager, networkId);
 
         store = serviceDirectory.get(VirtualNetworkFlowRuleStore.class);
+
         idGenerator = serviceDirectory.get(CoreService.class)
                 .getIdGenerator(VIRTUAL_FLOW_OP_TOPIC + networkId().toString());
         providerRegistryService =
@@ -116,7 +123,9 @@
         innerProviderService = new InternalFlowRuleProviderService();
         providerRegistryService.registerProviderService(networkId(), innerProviderService);
 
-        this.deviceService = manager.get(networkId(), DeviceService.class);
+        this.deviceService = manager.get(networkId, DeviceService.class);
+        this.storeDelegate = new InternalStoreDelegate();
+        store.setDelegate(networkId, this.storeDelegate);
     }
 
     @Override
@@ -288,13 +297,23 @@
         }
     }
 
-    private class InternalFlowRuleProviderService
+    private final class InternalFlowRuleProviderService
             extends AbstractVirtualProviderService<VirtualFlowRuleProvider>
             implements VirtualFlowRuleProviderService {
 
         final Map<FlowEntry, Long> firstSeen = Maps.newConcurrentMap();
         final Map<FlowEntry, Long> lastSeen = Maps.newConcurrentMap();
 
+        private InternalFlowRuleProviderService() {
+            //TODO: find a proper virtual provider.
+            Set<ProviderId> providerIds =
+                    providerRegistryService.getProvidersByService(this);
+            ProviderId providerId = providerIds.stream().findFirst().get();
+            VirtualFlowRuleProvider provider = (VirtualFlowRuleProvider)
+                    providerRegistryService.getProvider(providerId);
+            setProvider(provider);
+        }
+
         @Override
         public void flowRemoved(FlowEntry flowEntry) {
             checkNotNull(flowEntry, FLOW_RULE_NULL);
@@ -310,7 +329,6 @@
             if (flowEntry.reason() == FlowEntry.FlowRemoveReason.HARD_TIMEOUT) {
                 ((DefaultFlowEntry) stored).setState(FlowEntry.FlowEntryState.REMOVED);
             }
-            Device device = deviceService.getDevice(flowEntry.deviceId());
 
             //FIXME: obtains provider from devices providerId()
             FlowRuleEvent event = null;
@@ -333,7 +351,6 @@
             }
         }
 
-
         private void flowMissing(FlowEntry flowRule) {
             checkNotNull(flowRule, FLOW_RULE_NULL);
             checkValidity();
@@ -347,6 +364,7 @@
                 case ADDED:
                 case PENDING_ADD:
                     event = store.pendingFlowRule(networkId(), flowRule);
+
                     try {
                         provider().applyFlowRule(networkId(), flowRule);
                     } catch (UnsupportedOperationException e) {
@@ -494,4 +512,53 @@
             store.updateTableStatistics(networkId(), deviceId, tableStats);
         }
     }
+
+    // Store delegate to re-post events emitted from the store.
+    private class InternalStoreDelegate implements FlowRuleStoreDelegate {
+
+        // TODO: Right now we only dispatch events at individual flowEntry level.
+        // It may be more efficient for also dispatch events as a batch.
+        @Override
+        public void notify(FlowRuleBatchEvent event) {
+            final FlowRuleBatchRequest request = event.subject();
+            switch (event.type()) {
+                case BATCH_OPERATION_REQUESTED:
+                    // Request has been forwarded to MASTER Node, and was
+                    request.ops().forEach(
+                            op -> {
+                                switch (op.operator()) {
+                                    case ADD:
+                                        post(new FlowRuleEvent(RULE_ADD_REQUESTED, op.target()));
+                                        break;
+                                    case REMOVE:
+                                        post(new FlowRuleEvent(RULE_REMOVE_REQUESTED, op.target()));
+                                        break;
+                                    case MODIFY:
+                                        //TODO: do something here when the time comes.
+                                        break;
+                                    default:
+                                        log.warn("Unknown flow operation operator: {}", op.operator());
+                                }
+                            }
+                    );
+
+                    DeviceId deviceId = event.deviceId();
+                    FlowRuleBatchOperation batchOperation = request.asBatchOperation(deviceId);
+
+                    VirtualFlowRuleProvider provider = innerProviderService.provider();
+                    if (provider != null) {
+                        provider.executeBatch(networkId, batchOperation);
+                    }
+
+                    break;
+
+                case BATCH_OPERATION_COMPLETED:
+                    //TODO: do post-processing for batch operations.
+                    break;
+
+                default:
+                    break;
+            }
+        }
+    }
 }
diff --git a/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/provider/VirtualProviderManager.java b/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/provider/VirtualProviderManager.java
index 59c6630..40a791f 100644
--- a/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/provider/VirtualProviderManager.java
+++ b/incubator/net/src/main/java/org/onosproject/incubator/net/virtual/impl/provider/VirtualProviderManager.java
@@ -110,13 +110,14 @@
     }
 
     @Override
-    public Set<ProviderId> getProvidersByService(VirtualProviderService virtualProviderService) {
+    public Set<ProviderId> getProvidersByService(VirtualProviderService
+                                                             virtualProviderService) {
         Class clazz = getProviderClass(virtualProviderService);
 
         return ImmutableSet.copyOf(providers.values().stream()
-                                           .filter(p -> p.getClass()
-                                                   .isAssignableFrom(clazz))
-                                           .map(p -> p.id()).collect(Collectors.toSet()));
+                                           .filter(clazz::isInstance)
+                                           .map(VirtualProvider::id)
+                                           .collect(Collectors.toSet()));
     }
 
     @Override
@@ -144,8 +145,7 @@
         }
 
         return services.stream()
-                .filter(s -> virtualProvider.getClass()
-                        .isAssignableFrom(getProviderClass(s)))
+                .filter(s -> getProviderClass(s).isInstance(virtualProvider))
                 .findFirst().get();
     }
 
diff --git a/incubator/net/src/test/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowRuleManagerTest.java b/incubator/net/src/test/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowRuleManagerTest.java
index 593c6d6..40658bd 100644
--- a/incubator/net/src/test/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowRuleManagerTest.java
+++ b/incubator/net/src/test/java/org/onosproject/incubator/net/virtual/impl/VirtualNetworkFlowRuleManagerTest.java
@@ -16,6 +16,8 @@
 
 package org.onosproject.incubator.net.virtual.impl;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.MoreExecutors;
@@ -29,6 +31,7 @@
 import org.onosproject.common.event.impl.TestEventDispatcher;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.CoreService;
+import org.onosproject.event.EventDeliveryService;
 import org.onosproject.incubator.net.virtual.NetworkId;
 import org.onosproject.incubator.net.virtual.TenantId;
 import org.onosproject.incubator.net.virtual.VirtualDevice;
@@ -36,11 +39,15 @@
 import org.onosproject.incubator.net.virtual.VirtualNetwork;
 import org.onosproject.incubator.net.virtual.VirtualNetworkFlowRuleStore;
 import org.onosproject.incubator.net.virtual.VirtualNetworkStore;
+import org.onosproject.incubator.net.virtual.event.VirtualEvent;
+import org.onosproject.incubator.net.virtual.event.VirtualListenerRegistryManager;
 import org.onosproject.incubator.net.virtual.impl.provider.VirtualProviderManager;
 import org.onosproject.incubator.net.virtual.provider.AbstractVirtualProvider;
 import org.onosproject.incubator.net.virtual.provider.VirtualFlowRuleProvider;
+import org.onosproject.incubator.net.virtual.provider.VirtualFlowRuleProviderService;
 import org.onosproject.incubator.net.virtual.provider.VirtualProviderRegistryService;
 import org.onosproject.incubator.store.virtual.impl.DistributedVirtualNetworkStore;
+import org.onosproject.incubator.store.virtual.impl.SimpleVirtualFlowRuleStore;
 import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Link;
@@ -50,17 +57,12 @@
 import org.onosproject.net.flow.DefaultFlowEntry;
 import org.onosproject.net.flow.DefaultFlowRule;
 import org.onosproject.net.flow.FlowEntry;
-import org.onosproject.net.flow.FlowId;
 import org.onosproject.net.flow.FlowRule;
-import org.onosproject.net.flow.FlowRuleBatchEntry;
-import org.onosproject.net.flow.FlowRuleBatchEvent;
 import org.onosproject.net.flow.FlowRuleBatchOperation;
 import org.onosproject.net.flow.FlowRuleEvent;
 import org.onosproject.net.flow.FlowRuleListener;
 import org.onosproject.net.flow.FlowRuleService;
-import org.onosproject.net.flow.FlowRuleStoreDelegate;
 import org.onosproject.net.flow.StoredFlowEntry;
-import org.onosproject.net.flow.TableStatisticsEntry;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.criteria.Criterion;
@@ -73,13 +75,13 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 
 import static org.junit.Assert.*;
+import static org.onosproject.net.flow.FlowRuleEvent.Type.*;
 
 public class VirtualNetworkFlowRuleManagerTest extends TestDeviceParams {
     private static final int TIMEOUT = 10;
@@ -89,12 +91,18 @@
     private TestableIntentService intentService = new FakeIntentManager();
     private ServiceDirectory testDirectory;
     private VirtualNetworkFlowRuleStore flowRuleStore;
-    private VirtualProviderRegistryService providerRegistryService;
+    private VirtualProviderManager providerRegistryService;
+
+    private EventDeliveryService eventDeliveryService;
+    VirtualListenerRegistryManager listenerRegistryManager =
+            VirtualListenerRegistryManager.getInstance();
 
     private VirtualNetworkFlowRuleManager vnetFlowRuleService1;
     private VirtualNetworkFlowRuleManager vnetFlowRuleService2;
 
     private VirtualFlowRuleProvider provider = new TestProvider();
+    private VirtualFlowRuleProviderService providerService1;
+    private VirtualFlowRuleProviderService providerService2;
 
     protected TestFlowRuleListener listener1 = new TestFlowRuleListener();
     protected TestFlowRuleListener listener2 = new TestFlowRuleListener();
@@ -116,7 +124,7 @@
         TestUtils.setField(virtualNetworkManagerStore, "storageService", new TestStorageService());
         virtualNetworkManagerStore.activate();
 
-        flowRuleStore = new TestVirtualFlowRuleStore();
+        flowRuleStore = new SimpleVirtualFlowRuleStore();
 
         providerRegistryService = new VirtualProviderManager();
         providerRegistryService.registerProvider(provider);
@@ -125,7 +133,10 @@
         manager.store = virtualNetworkManagerStore;
         manager.intentService = intentService;
         TestUtils.setField(manager, "coreService", coreService);
-        NetTestTools.injectEventDispatcher(manager, new TestEventDispatcher());
+
+        eventDeliveryService = new TestEventDispatcher();
+        NetTestTools.injectEventDispatcher(manager, eventDeliveryService);
+        eventDeliveryService.addSink(VirtualEvent.class, listenerRegistryManager);
 
         appId = new TestApplicationId("FlowRuleManagerTest");
 
@@ -133,6 +144,7 @@
                 .add(VirtualNetworkStore.class, virtualNetworkManagerStore)
                 .add(CoreService.class, coreService)
                 .add(VirtualProviderRegistryService.class, providerRegistryService)
+                .add(EventDeliveryService.class, eventDeliveryService)
                 .add(VirtualNetworkFlowRuleStore.class, flowRuleStore);
         TestUtils.setField(manager, "serviceDirectory", testDirectory);
 
@@ -144,11 +156,17 @@
         vnetFlowRuleService1 = new VirtualNetworkFlowRuleManager(manager, vnet1.id());
         vnetFlowRuleService2 = new VirtualNetworkFlowRuleManager(manager, vnet2.id());
         vnetFlowRuleService1.addListener(listener1);
+        vnetFlowRuleService2.addListener(listener2);
 
         vnetFlowRuleService1.operationsService = MoreExecutors.newDirectExecutorService();
         vnetFlowRuleService2.operationsService = MoreExecutors.newDirectExecutorService();
         vnetFlowRuleService1.deviceInstallers = MoreExecutors.newDirectExecutorService();
         vnetFlowRuleService2.deviceInstallers = MoreExecutors.newDirectExecutorService();
+
+        providerService1 = (VirtualFlowRuleProviderService)
+                providerRegistryService.getProviderService(vnet1.id(), provider);
+        providerService2 = (VirtualFlowRuleProviderService)
+                providerRegistryService.getProviderService(vnet2.id(), provider);
     }
 
     @After
@@ -246,6 +264,7 @@
                    Sets.newHashSet(vnetFlowRuleService1.getFlowEntries(DID1)).isEmpty());
         assertTrue("store should be empty",
                    Sets.newHashSet(vnetFlowRuleService2.getFlowEntries(DID1)).isEmpty());
+
         FlowRule f1 = addFlowRule(1);
         FlowRule f2 = addFlowRule(2);
 
@@ -254,6 +273,180 @@
 
         assertEquals("2 rules should exist", 2, flowCount(vnetFlowRuleService1));
         assertEquals("0 rules should exist", 0, flowCount(vnetFlowRuleService2));
+
+        providerService1.pushFlowMetrics(DID1, ImmutableList.of(fe1, fe2));
+        validateEvents(listener1, RULE_ADD_REQUESTED, RULE_ADD_REQUESTED,
+                       RULE_ADDED, RULE_ADDED);
+
+        addFlowRule(1);
+        assertEquals("should still be 2 rules", 2, flowCount(vnetFlowRuleService1));
+        System.err.println("events :" + listener1.events);
+        assertEquals("0 rules should exist", 0, flowCount(vnetFlowRuleService2));
+
+        providerService1.pushFlowMetrics(DID1, ImmutableList.of(fe1));
+        validateEvents(listener1, RULE_UPDATED, RULE_UPDATED);
+    }
+
+    @Test
+    public void applyFlowRules() {
+        FlowRule r1 = flowRule(1, 1);
+        FlowRule r2 = flowRule(2, 2);
+        FlowRule r3 = flowRule(3, 3);
+
+        assertTrue("store should be empty",
+                   Sets.newHashSet(vnetFlowRuleService1.getFlowEntries(DID1)).isEmpty());
+        vnetFlowRuleService1.applyFlowRules(r1, r2, r3);
+        assertEquals("3 rules should exist", 3, flowCount(vnetFlowRuleService1));
+        assertTrue("Entries should be pending add.",
+                   validateState(ImmutableMap.of(
+                           r1, FlowEntry.FlowEntryState.PENDING_ADD,
+                           r2, FlowEntry.FlowEntryState.PENDING_ADD,
+                           r3, FlowEntry.FlowEntryState.PENDING_ADD)));
+    }
+
+    @Test
+    public void purgeFlowRules() {
+        FlowRule f1 = addFlowRule(1);
+        FlowRule f2 = addFlowRule(2);
+        FlowRule f3 = addFlowRule(3);
+        assertEquals("3 rules should exist", 3, flowCount(vnetFlowRuleService1));
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
+        FlowEntry fe3 = new DefaultFlowEntry(f3);
+        providerService1.pushFlowMetrics(DID1, ImmutableList.of(fe1, fe2, fe3));
+        validateEvents(listener1, RULE_ADD_REQUESTED, RULE_ADD_REQUESTED, RULE_ADD_REQUESTED,
+                       RULE_ADDED, RULE_ADDED, RULE_ADDED);
+        vnetFlowRuleService1.purgeFlowRules(DID1);
+        assertEquals("0 rule should exist", 0, flowCount(vnetFlowRuleService1));
+    }
+
+    @Test
+    public void removeFlowRules() {
+        FlowRule f1 = addFlowRule(1);
+        FlowRule f2 = addFlowRule(2);
+        FlowRule f3 = addFlowRule(3);
+        assertEquals("3 rules should exist", 3, flowCount(vnetFlowRuleService1));
+
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
+        FlowEntry fe3 = new DefaultFlowEntry(f3);
+        providerService1.pushFlowMetrics(DID1, ImmutableList.of(fe1, fe2, fe3));
+        validateEvents(listener1, RULE_ADD_REQUESTED, RULE_ADD_REQUESTED, RULE_ADD_REQUESTED,
+                       RULE_ADDED, RULE_ADDED, RULE_ADDED);
+
+        vnetFlowRuleService1.removeFlowRules(f1, f2);
+        //removing from north, so no events generated
+        validateEvents(listener1, RULE_REMOVE_REQUESTED, RULE_REMOVE_REQUESTED);
+        assertEquals("3 rule should exist", 3, flowCount(vnetFlowRuleService1));
+        assertTrue("Entries should be pending remove.",
+                   validateState(ImmutableMap.of(
+                           f1, FlowEntry.FlowEntryState.PENDING_REMOVE,
+                           f2, FlowEntry.FlowEntryState.PENDING_REMOVE,
+                           f3, FlowEntry.FlowEntryState.ADDED)));
+
+        vnetFlowRuleService1.removeFlowRules(f1);
+        assertEquals("3 rule should still exist", 3, flowCount(vnetFlowRuleService1));
+    }
+
+    @Test
+    public void flowRemoved() {
+        FlowRule f1 = addFlowRule(1);
+        FlowRule f2 = addFlowRule(2);
+        StoredFlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
+
+        providerService1.pushFlowMetrics(DID1, ImmutableList.of(fe1, fe2));
+        vnetFlowRuleService1.removeFlowRules(f1);
+
+        //FIXME modification of "stored" flow entry outside of store
+        fe1.setState(FlowEntry.FlowEntryState.REMOVED);
+
+        providerService1.flowRemoved(fe1);
+
+        validateEvents(listener1, RULE_ADD_REQUESTED, RULE_ADD_REQUESTED, RULE_ADDED,
+                       RULE_ADDED, RULE_REMOVE_REQUESTED, RULE_REMOVED);
+
+        providerService1.flowRemoved(fe1);
+        validateEvents(listener1);
+
+        FlowRule f3 = flowRule(3, 3);
+        FlowEntry fe3 = new DefaultFlowEntry(f3);
+        vnetFlowRuleService1.applyFlowRules(f3);
+
+        providerService1.pushFlowMetrics(DID1, Collections.singletonList(fe3));
+        validateEvents(listener1, RULE_ADD_REQUESTED, RULE_ADDED, RULE_UPDATED);
+
+        providerService1.flowRemoved(fe3);
+        validateEvents(listener1);
+    }
+
+    @Test
+    public void extraneousFlow() {
+        FlowRule f1 = flowRule(1, 1);
+        FlowRule f2 = flowRule(2, 2);
+        FlowRule f3 = flowRule(3, 3);
+        vnetFlowRuleService1.applyFlowRules(f1, f2);
+
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
+        FlowEntry fe3 = new DefaultFlowEntry(f3);
+
+
+        providerService1.pushFlowMetrics(DID1, Lists.newArrayList(fe1, fe2, fe3));
+
+        validateEvents(listener1, RULE_ADD_REQUESTED, RULE_ADD_REQUESTED,
+                       RULE_ADDED, RULE_ADDED);
+    }
+
+    /*
+     * Tests whether a rule that was marked for removal but no flowRemoved was received
+     * is indeed removed at the next stats update.
+     */
+    @Test
+    public void flowMissingRemove() {
+        FlowRule f1 = flowRule(1, 1);
+        FlowRule f2 = flowRule(2, 2);
+        FlowRule f3 = flowRule(3, 3);
+
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
+        vnetFlowRuleService1.applyFlowRules(f1, f2, f3);
+
+        vnetFlowRuleService1.removeFlowRules(f3);
+
+        providerService1.pushFlowMetrics(DID1, Lists.newArrayList(fe1, fe2));
+
+        validateEvents(listener1, RULE_ADD_REQUESTED, RULE_ADD_REQUESTED, RULE_ADD_REQUESTED,
+                       RULE_REMOVE_REQUESTED, RULE_ADDED, RULE_ADDED, RULE_REMOVED);
+    }
+
+    @Test
+    public void removeByAppId() {
+        FlowRule f1 = flowRule(1, 1);
+        FlowRule f2 = flowRule(2, 2);
+        vnetFlowRuleService1.applyFlowRules(f1, f2);
+
+        vnetFlowRuleService1.removeFlowRulesById(appId);
+
+        //only check that we are in pending remove. Events and actual remove state will
+        // be set by flowRemoved call.
+        validateState(ImmutableMap.of(
+                f1, FlowEntry.FlowEntryState.PENDING_REMOVE,
+                f2, FlowEntry.FlowEntryState.PENDING_REMOVE));
+    }
+
+    //TODO:Tests for fallback
+
+    private boolean validateState(Map<FlowRule, FlowEntry.FlowEntryState> expected) {
+        Map<FlowRule, FlowEntry.FlowEntryState> expectedToCheck = new HashMap<>(expected);
+        Iterable<FlowEntry> rules = vnetFlowRuleService1.getFlowEntries(DID1);
+        for (FlowEntry f : rules) {
+            assertTrue("Unexpected FlowRule " + f, expectedToCheck.containsKey(f));
+            assertEquals("FlowEntry" + f, expectedToCheck.get(f), f.state());
+            expectedToCheck.remove(f);
+        }
+        assertEquals(Collections.emptySet(), expectedToCheck.entrySet());
+        return true;
     }
 
     private class TestSelector implements TrafficSelector {
@@ -288,7 +481,6 @@
             }
             return false;
         }
-
     }
 
     private class TestTreatment implements TrafficTreatment {
@@ -349,118 +541,6 @@
         }
     }
 
-    private class TestVirtualFlowRuleStore implements VirtualNetworkFlowRuleStore {
-
-        private final ConcurrentMap<NetworkId,
-                ConcurrentMap<DeviceId, ConcurrentMap<FlowId, List<StoredFlowEntry>>>>
-                flowEntries = new ConcurrentHashMap<>();
-
-        @Override
-        public void setDelegate(FlowRuleStoreDelegate delegate) {
-
-        }
-
-        @Override
-        public void unsetDelegate(FlowRuleStoreDelegate delegate) {
-
-        }
-
-        @Override
-        public boolean hasDelegate() {
-            return false;
-        }
-
-        @Override
-        public int getFlowRuleCount(NetworkId networkId) {
-            return 0;
-        }
-
-        @Override
-        public FlowEntry getFlowEntry(NetworkId networkId, FlowRule rule) {
-            return null;
-        }
-
-        @Override
-        public Iterable<FlowEntry> getFlowEntries(NetworkId networkId, DeviceId deviceId) {
-            HashSet<FlowEntry> entries = Sets.newHashSet();
-
-            if (flowEntries.get(networkId) == null
-                    || flowEntries.get(networkId).get(deviceId) == null) {
-                return entries;
-            }
-
-            flowEntries.get(networkId).get(deviceId).values().forEach(e -> entries.addAll(e));
-
-            return entries;
-        }
-
-        @Override
-        public void storeFlowRule(NetworkId networkId, FlowRule rule) {
-            StoredFlowEntry entry = new DefaultFlowEntry(rule);
-            flowEntries.putIfAbsent(networkId, new ConcurrentHashMap<>());
-            flowEntries.get(networkId).putIfAbsent(rule.deviceId(), new ConcurrentHashMap<>());
-            flowEntries.get(networkId).get(rule.deviceId()).putIfAbsent(rule.id(), Lists.newArrayList());
-            flowEntries.get(networkId).get(rule.deviceId()).get(rule.id()).add(entry);
-        }
-
-        @Override
-        public void storeBatch(NetworkId networkId, FlowRuleBatchOperation batchOperation) {
-            for (FlowRuleBatchEntry entry : batchOperation.getOperations()) {
-                final FlowRule flowRule = entry.target();
-                if (entry.operator().equals(FlowRuleBatchEntry.FlowRuleOperation.ADD)) {
-                    storeFlowRule(networkId, flowRule);
-                } else if (entry.operator().equals(FlowRuleBatchEntry.FlowRuleOperation.REMOVE)) {
-                    deleteFlowRule(networkId, flowRule);
-                } else {
-                    throw new UnsupportedOperationException("Unsupported operation type");
-                }
-            }
-        }
-
-        @Override
-        public void batchOperationComplete(NetworkId networkId, FlowRuleBatchEvent event) {
-
-        }
-
-        @Override
-        public void deleteFlowRule(NetworkId networkId, FlowRule rule) {
-
-        }
-
-        @Override
-        public FlowRuleEvent addOrUpdateFlowRule(NetworkId networkId, FlowEntry rule) {
-            return null;
-        }
-
-        @Override
-        public FlowRuleEvent removeFlowRule(NetworkId networkId, FlowEntry rule) {
-            return null;
-        }
-
-        @Override
-        public FlowRuleEvent pendingFlowRule(NetworkId networkId, FlowEntry rule) {
-            return null;
-        }
-
-        @Override
-        public void purgeFlowRules(NetworkId networkId) {
-
-        }
-
-        @Override
-        public FlowRuleEvent updateTableStatistics(NetworkId networkId,
-                                                   DeviceId deviceId,
-                                                   List<TableStatisticsEntry> tableStats) {
-            return null;
-        }
-
-        @Override
-        public Iterable<TableStatisticsEntry>
-        getTableStatistics(NetworkId networkId, DeviceId deviceId) {
-            return null;
-        }
-    }
-
     private void validateEvents(TestFlowRuleListener listener, FlowRuleEvent.Type... events) {
         if (events == null) {
             assertTrue("events generated", listener.events.isEmpty());
@@ -485,7 +565,7 @@
 
         @Override
         public void event(FlowRuleEvent event) {
-           events.add(event);
+            events.add(event);
         }
     }