DistributedDeviceStore

Change-Id: I34cf5a787bf0f9d16840bf2e3cc8d0167060f628
diff --git a/core/store/src/main/java/org/onlab/onos/store/device/impl/DistributedDeviceStore.java b/core/store/src/main/java/org/onlab/onos/store/device/impl/DistributedDeviceStore.java
index 62a4c5e..3186578 100644
--- a/core/store/src/main/java/org/onlab/onos/store/device/impl/DistributedDeviceStore.java
+++ b/core/store/src/main/java/org/onlab/onos/store/device/impl/DistributedDeviceStore.java
@@ -369,7 +369,7 @@
         }
 
         @Override
-        protected void onUpdate(DeviceId deviceId, DefaultDevice device) {
+        protected void onUpdate(DeviceId deviceId, DefaultDevice oldDevice, DefaultDevice device) {
             notifyDelegate(new DeviceEvent(DEVICE_UPDATED, device));
         }
     }
@@ -390,7 +390,7 @@
         }
 
         @Override
-        protected void onUpdate(DeviceId deviceId, Map<PortNumber, Port> ports) {
+        protected void onUpdate(DeviceId deviceId, Map<PortNumber, Port> oldPorts, Map<PortNumber, Port> ports) {
 //            notifyDelegate(new DeviceEvent(PORT_UPDATED, getDevice(deviceId)));
         }
     }
diff --git a/core/store/src/main/java/org/onlab/onos/store/impl/AbstractDistributedStore.java b/core/store/src/main/java/org/onlab/onos/store/impl/AbstractDistributedStore.java
index 41af9b3..eb795a8 100644
--- a/core/store/src/main/java/org/onlab/onos/store/impl/AbstractDistributedStore.java
+++ b/core/store/src/main/java/org/onlab/onos/store/impl/AbstractDistributedStore.java
@@ -101,7 +101,7 @@
             V newVal = deserialize(event.getValue());
             Optional<V> newValue = Optional.of(newVal);
             cache.asMap().replace(key, oldValue, newValue);
-            onUpdate(key, newVal);
+            onUpdate(key, oldVal, newVal);
         }
 
         @Override
@@ -125,9 +125,10 @@
          * Cache entry update hook.
          *
          * @param key    new key
+         * @param oldValue old value
          * @param newVal new value
          */
-        protected void onUpdate(K key, V newVal) {
+        protected void onUpdate(K key, V oldValue, V newVal) {
         }
 
         /**
diff --git a/core/store/src/main/java/org/onlab/onos/store/impl/StoreManager.java b/core/store/src/main/java/org/onlab/onos/store/impl/StoreManager.java
index c22d915..e2692d5 100644
--- a/core/store/src/main/java/org/onlab/onos/store/impl/StoreManager.java
+++ b/core/store/src/main/java/org/onlab/onos/store/impl/StoreManager.java
@@ -14,19 +14,26 @@
 import org.onlab.onos.cluster.ControllerNode;
 import org.onlab.onos.cluster.DefaultControllerNode;
 import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.DefaultDevice;
+import org.onlab.onos.net.DefaultLink;
 import org.onlab.onos.net.DefaultPort;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Element;
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.LinkKey;
 import org.onlab.onos.net.MastershipRole;
 import org.onlab.onos.net.Port;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.common.StoreService;
+import org.onlab.onos.store.serializers.ConnectPointSerializer;
+import org.onlab.onos.store.serializers.DefaultLinkSerializer;
 import org.onlab.onos.store.serializers.DefaultPortSerializer;
 import org.onlab.onos.store.serializers.DeviceIdSerializer;
 import org.onlab.onos.store.serializers.IpPrefixSerializer;
+import org.onlab.onos.store.serializers.LinkKeySerializer;
 import org.onlab.onos.store.serializers.NodeIdSerializer;
 import org.onlab.onos.store.serializers.OnosTimestampSerializer;
 import org.onlab.onos.store.serializers.PortNumberSerializer;
@@ -84,7 +91,9 @@
                           DefaultDevice.class,
                           MastershipRole.class,
                           Port.class,
-                          Element.class
+                          Element.class,
+
+                          Link.Type.class
                 )
                 .register(IpPrefix.class, new IpPrefixSerializer())
                 .register(URI.class, new URISerializer())
@@ -94,6 +103,9 @@
                 .register(PortNumber.class, new PortNumberSerializer())
                 .register(DefaultPort.class, new DefaultPortSerializer())
                 .register(OnosTimestamp.class, new OnosTimestampSerializer())
+                .register(LinkKey.class, new LinkKeySerializer())
+                .register(ConnectPoint.class, new ConnectPointSerializer())
+                .register(DefaultLink.class, new DefaultLinkSerializer())
                 .build()
                 .populate(10);
     }
diff --git a/core/store/src/main/java/org/onlab/onos/store/link/impl/DistributedLinkStore.java b/core/store/src/main/java/org/onlab/onos/store/link/impl/DistributedLinkStore.java
new file mode 100644
index 0000000..6db9695
--- /dev/null
+++ b/core/store/src/main/java/org/onlab/onos/store/link/impl/DistributedLinkStore.java
@@ -0,0 +1,258 @@
+package org.onlab.onos.store.link.impl;
+
+import static com.google.common.cache.CacheBuilder.newBuilder;
+import static org.onlab.onos.net.Link.Type.DIRECT;
+import static org.onlab.onos.net.Link.Type.INDIRECT;
+import static org.onlab.onos.net.link.LinkEvent.Type.LINK_ADDED;
+import static org.onlab.onos.net.link.LinkEvent.Type.LINK_REMOVED;
+import static org.onlab.onos.net.link.LinkEvent.Type.LINK_UPDATED;
+import static org.slf4j.LoggerFactory.getLogger;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DefaultLink;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.LinkKey;
+import org.onlab.onos.net.link.LinkDescription;
+import org.onlab.onos.net.link.LinkEvent;
+import org.onlab.onos.net.link.LinkStore;
+import org.onlab.onos.net.link.LinkStoreDelegate;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.impl.AbsentInvalidatingLoadingCache;
+import org.onlab.onos.store.impl.AbstractDistributedStore;
+import org.onlab.onos.store.impl.OptionalCacheLoader;
+import org.slf4j.Logger;
+
+import com.google.common.base.Optional;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.ImmutableSet.Builder;
+import com.hazelcast.core.IMap;
+
+/**
+ * Manages inventory of infrastructure links using Hazelcast-backed map.
+ */
+@Component(immediate = true)
+@Service
+public class DistributedLinkStore
+    extends AbstractDistributedStore<LinkEvent, LinkStoreDelegate>
+    implements LinkStore {
+
+    private final Logger log = getLogger(getClass());
+
+    // Link inventory
+    private IMap<byte[], byte[]> rawLinks;
+    private LoadingCache<LinkKey, Optional<DefaultLink>> links;
+
+    // TODO synchronize?
+    // Egress and ingress link sets
+    private final Multimap<DeviceId, Link> srcLinks = HashMultimap.create();
+    private final Multimap<DeviceId, Link> dstLinks = HashMultimap.create();
+
+    @Override
+    @Activate
+    public void activate() {
+        super.activate();
+
+        boolean includeValue = true;
+
+        // TODO decide on Map name scheme to avoid collision
+        rawLinks = theInstance.getMap("links");
+        final OptionalCacheLoader<LinkKey, DefaultLink> linkLoader
+                = new OptionalCacheLoader<>(storeService, rawLinks);
+        links = new AbsentInvalidatingLoadingCache<>(newBuilder().build(linkLoader));
+        // refresh/populate cache based on notification from other instance
+        rawLinks.addEntryListener(new RemoteLinkEventHandler(links), includeValue);
+
+        loadLinkCache();
+
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        super.activate();
+        log.info("Stopped");
+    }
+
+    private void loadLinkCache() {
+        for (byte[] keyBytes : rawLinks.keySet()) {
+            final LinkKey id = deserialize(keyBytes);
+            links.refresh(id);
+        }
+    }
+
+    @Override
+    public int getLinkCount() {
+        return links.asMap().size();
+    }
+
+    @Override
+    public Iterable<Link> getLinks() {
+        Builder<Link> builder = ImmutableSet.builder();
+        for (Optional<DefaultLink> e : links.asMap().values()) {
+            if (e.isPresent()) {
+                builder.add(e.get());
+            }
+        }
+        return builder.build();
+    }
+
+    @Override
+    public Set<Link> getDeviceEgressLinks(DeviceId deviceId) {
+        return ImmutableSet.copyOf(srcLinks.get(deviceId));
+    }
+
+    @Override
+    public Set<Link> getDeviceIngressLinks(DeviceId deviceId) {
+        return ImmutableSet.copyOf(dstLinks.get(deviceId));
+    }
+
+    @Override
+    public Link getLink(ConnectPoint src, ConnectPoint dst) {
+        return links.getUnchecked(new LinkKey(src, dst)).orNull();
+    }
+
+    @Override
+    public Set<Link> getEgressLinks(ConnectPoint src) {
+        Set<Link> egress = new HashSet<>();
+        for (Link link : srcLinks.get(src.deviceId())) {
+            if (link.src().equals(src)) {
+                egress.add(link);
+            }
+        }
+        return egress;
+    }
+
+    @Override
+    public Set<Link> getIngressLinks(ConnectPoint dst) {
+        Set<Link> ingress = new HashSet<>();
+        for (Link link : dstLinks.get(dst.deviceId())) {
+            if (link.dst().equals(dst)) {
+                ingress.add(link);
+            }
+        }
+        return ingress;
+    }
+
+    @Override
+    public LinkEvent createOrUpdateLink(ProviderId providerId,
+                                        LinkDescription linkDescription) {
+        LinkKey key = new LinkKey(linkDescription.src(), linkDescription.dst());
+        Optional<DefaultLink> link = links.getUnchecked(key);
+        if (!link.isPresent()) {
+            return createLink(providerId, key, linkDescription);
+        }
+        return updateLink(providerId, link.get(), key, linkDescription);
+    }
+
+    // Creates and stores the link and returns the appropriate event.
+    private LinkEvent createLink(ProviderId providerId, LinkKey key,
+                                 LinkDescription linkDescription) {
+        DefaultLink link = new DefaultLink(providerId, key.src(), key.dst(),
+                                           linkDescription.type());
+        synchronized (this) {
+            final byte[] keyBytes = serialize(key);
+            rawLinks.put(keyBytes, serialize(link));
+            links.asMap().putIfAbsent(key, Optional.of(link));
+
+            addNewLink(link);
+        }
+        return new LinkEvent(LINK_ADDED, link);
+    }
+
+    // update Egress and ingress link sets
+    private void addNewLink(DefaultLink link) {
+        synchronized (this) {
+            srcLinks.put(link.src().deviceId(), link);
+            dstLinks.put(link.dst().deviceId(), link);
+        }
+    }
+
+    // Updates, if necessary the specified link and returns the appropriate event.
+    private LinkEvent updateLink(ProviderId providerId, DefaultLink link,
+                                 LinkKey key, LinkDescription linkDescription) {
+        // FIXME confirm Link update condition is OK
+        if (link.type() == INDIRECT && linkDescription.type() == DIRECT) {
+            synchronized (this) {
+
+                DefaultLink updated =
+                        new DefaultLink(providerId, link.src(), link.dst(),
+                                        linkDescription.type());
+                final byte[] keyBytes = serialize(key);
+                rawLinks.put(keyBytes, serialize(updated));
+                links.asMap().replace(key, Optional.of(link), Optional.of(updated));
+
+                replaceLink(link, updated);
+                return new LinkEvent(LINK_UPDATED, updated);
+            }
+        }
+        return null;
+    }
+
+    // update Egress and ingress link sets
+    private void replaceLink(DefaultLink link, DefaultLink updated) {
+        synchronized (this) {
+            srcLinks.remove(link.src().deviceId(), link);
+            dstLinks.remove(link.dst().deviceId(), link);
+
+            srcLinks.put(link.src().deviceId(), updated);
+            dstLinks.put(link.dst().deviceId(), updated);
+        }
+    }
+
+    @Override
+    public LinkEvent removeLink(ConnectPoint src, ConnectPoint dst) {
+        synchronized (this) {
+            LinkKey key = new LinkKey(src, dst);
+            byte[] keyBytes = serialize(key);
+            Link link = deserialize(rawLinks.remove(keyBytes));
+            links.invalidate(key);
+            if (link != null) {
+                removeLink(link);
+                return new LinkEvent(LINK_REMOVED, link);
+            }
+            return null;
+        }
+    }
+
+    // update Egress and ingress link sets
+    private void removeLink(Link link) {
+        synchronized (this) {
+            srcLinks.remove(link.src().deviceId(), link);
+            dstLinks.remove(link.dst().deviceId(), link);
+        }
+    }
+
+    private class RemoteLinkEventHandler extends RemoteEventHandler<LinkKey, DefaultLink> {
+        public RemoteLinkEventHandler(LoadingCache<LinkKey, Optional<DefaultLink>> cache) {
+            super(cache);
+        }
+
+        @Override
+        protected void onAdd(LinkKey key, DefaultLink newVal) {
+            addNewLink(newVal);
+            notifyDelegate(new LinkEvent(LINK_ADDED, newVal));
+        }
+
+        @Override
+        protected void onUpdate(LinkKey key, DefaultLink oldVal, DefaultLink newVal) {
+            replaceLink(oldVal, newVal);
+            notifyDelegate(new LinkEvent(LINK_UPDATED, newVal));
+        }
+
+        @Override
+        protected void onRemove(LinkKey key, DefaultLink val) {
+            removeLink(val);
+            notifyDelegate(new LinkEvent(LINK_REMOVED, val));
+        }
+    }
+}
diff --git a/core/store/src/main/java/org/onlab/onos/store/link/impl/package-info.java b/core/store/src/main/java/org/onlab/onos/store/link/impl/package-info.java
new file mode 100644
index 0000000..53b6f33
--- /dev/null
+++ b/core/store/src/main/java/org/onlab/onos/store/link/impl/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * Implementation of link store using Hazelcast distributed structures.
+ */
+package org.onlab.onos.store.link.impl;
diff --git a/core/store/src/main/java/org/onlab/onos/store/serializers/ConnectPointSerializer.java b/core/store/src/main/java/org/onlab/onos/store/serializers/ConnectPointSerializer.java
new file mode 100644
index 0000000..46badcb
--- /dev/null
+++ b/core/store/src/main/java/org/onlab/onos/store/serializers/ConnectPointSerializer.java
@@ -0,0 +1,37 @@
+package org.onlab.onos.store.serializers;
+
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.ElementId;
+import org.onlab.onos.net.PortNumber;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Kryo Serializer for {@link ConnectPointSerializer}.
+ */
+public class ConnectPointSerializer extends Serializer<ConnectPoint> {
+
+    /**
+     * Default constructor.
+     */
+    public ConnectPointSerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output, ConnectPoint object) {
+        kryo.writeClassAndObject(output, object.elementId());
+        kryo.writeClassAndObject(output, object.port());
+    }
+
+    @Override
+    public ConnectPoint read(Kryo kryo, Input input, Class<ConnectPoint> type) {
+        ElementId elementId = (ElementId) kryo.readClassAndObject(input);
+        PortNumber portNumber = (PortNumber) kryo.readClassAndObject(input);
+        return new ConnectPoint(elementId, portNumber);
+    }
+}
diff --git a/core/store/src/main/java/org/onlab/onos/store/serializers/DefaultLinkSerializer.java b/core/store/src/main/java/org/onlab/onos/store/serializers/DefaultLinkSerializer.java
new file mode 100644
index 0000000..5ee273d
--- /dev/null
+++ b/core/store/src/main/java/org/onlab/onos/store/serializers/DefaultLinkSerializer.java
@@ -0,0 +1,42 @@
+package org.onlab.onos.store.serializers;
+
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DefaultLink;
+import org.onlab.onos.net.Link.Type;
+import org.onlab.onos.net.provider.ProviderId;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Kryo Serializer for {@link DefaultLink}.
+ */
+public class DefaultLinkSerializer extends Serializer<DefaultLink> {
+
+    /**
+     * Default constructor.
+     */
+    public DefaultLinkSerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output, DefaultLink object) {
+        kryo.writeClassAndObject(output, object.providerId());
+        kryo.writeClassAndObject(output, object.src());
+        kryo.writeClassAndObject(output, object.dst());
+        kryo.writeClassAndObject(output, object.type());
+    }
+
+    @Override
+    public DefaultLink read(Kryo kryo, Input input, Class<DefaultLink> type) {
+        ProviderId providerId = (ProviderId) kryo.readClassAndObject(input);
+        ConnectPoint src = (ConnectPoint) kryo.readClassAndObject(input);
+        ConnectPoint dst = (ConnectPoint) kryo.readClassAndObject(input);
+        Type linkType = (Type) kryo.readClassAndObject(input);
+        return new DefaultLink(providerId, src, dst, linkType);
+    }
+}
diff --git a/core/store/src/main/java/org/onlab/onos/store/serializers/LinkKeySerializer.java b/core/store/src/main/java/org/onlab/onos/store/serializers/LinkKeySerializer.java
new file mode 100644
index 0000000..f635f3c
--- /dev/null
+++ b/core/store/src/main/java/org/onlab/onos/store/serializers/LinkKeySerializer.java
@@ -0,0 +1,35 @@
+package org.onlab.onos.store.serializers;
+
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.LinkKey;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Kryo Serializer for {@link LinkKey}.
+ */
+public class LinkKeySerializer extends Serializer<LinkKey> {
+
+    /**
+     * Default constructor.
+     */
+    public LinkKeySerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output, LinkKey object) {
+        kryo.writeClassAndObject(output, object.src());
+        kryo.writeClassAndObject(output, object.dst());
+    }
+
+    @Override
+    public LinkKey read(Kryo kryo, Input input, Class<LinkKey> type) {
+        ConnectPoint src = (ConnectPoint) kryo.readClassAndObject(input);
+        ConnectPoint dst = (ConnectPoint) kryo.readClassAndObject(input);
+        return new LinkKey(src, dst);
+    }
+}
diff --git a/core/store/src/test/java/org/onlab/onos/store/link/impl/DistributedLinkStoreTest.java b/core/store/src/test/java/org/onlab/onos/store/link/impl/DistributedLinkStoreTest.java
new file mode 100644
index 0000000..41853f6
--- /dev/null
+++ b/core/store/src/test/java/org/onlab/onos/store/link/impl/DistributedLinkStoreTest.java
@@ -0,0 +1,361 @@
+package org.onlab.onos.store.link.impl;
+
+import static org.junit.Assert.*;
+import static org.onlab.onos.net.DeviceId.deviceId;
+import static org.onlab.onos.net.Link.Type.*;
+import static org.onlab.onos.net.link.LinkEvent.Type.*;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.LinkKey;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.Link.Type;
+import org.onlab.onos.net.link.DefaultLinkDescription;
+import org.onlab.onos.net.link.LinkEvent;
+import org.onlab.onos.net.link.LinkStoreDelegate;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.common.StoreService;
+import org.onlab.onos.store.impl.StoreManager;
+import org.onlab.onos.store.impl.TestStoreManager;
+
+import com.google.common.collect.Iterables;
+import com.hazelcast.config.Config;
+import com.hazelcast.core.Hazelcast;
+
+public class DistributedLinkStoreTest {
+
+    private static final ProviderId PID = new ProviderId("of", "foo");
+    private static final DeviceId DID1 = deviceId("of:foo");
+    private static final DeviceId DID2 = deviceId("of:bar");
+//    private static final String MFR = "whitebox";
+//    private static final String HW = "1.1.x";
+//    private static final String SW1 = "3.8.1";
+//    private static final String SW2 = "3.9.5";
+//    private static final String SN = "43311-12345";
+
+    private static final PortNumber P1 = PortNumber.portNumber(1);
+    private static final PortNumber P2 = PortNumber.portNumber(2);
+    private static final PortNumber P3 = PortNumber.portNumber(3);
+
+    private StoreManager storeManager;
+
+    private DistributedLinkStore linkStore;
+
+    @BeforeClass
+    public static void setUpBeforeClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownAfterClass() throws Exception {
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        // TODO should find a way to clean Hazelcast instance without shutdown.
+        Config config = TestStoreManager.getTestConfig();
+
+        storeManager = new TestStoreManager(Hazelcast.newHazelcastInstance(config));
+        storeManager.activate();
+
+        linkStore = new TestDistributedLinkStore(storeManager);
+        linkStore.activate();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        linkStore.deactivate();
+        storeManager.deactivate();
+    }
+
+    private void putLink(DeviceId srcId, PortNumber srcNum,
+                         DeviceId dstId, PortNumber dstNum, Type type) {
+        ConnectPoint src = new ConnectPoint(srcId, srcNum);
+        ConnectPoint dst = new ConnectPoint(dstId, dstNum);
+        linkStore.createOrUpdateLink(PID, new DefaultLinkDescription(src, dst, type));
+    }
+
+    private void putLink(LinkKey key, Type type) {
+        putLink(key.src().deviceId(), key.src().port(),
+                key.dst().deviceId(), key.dst().port(),
+                type);
+    }
+
+    private static void assertLink(DeviceId srcId, PortNumber srcNum,
+                            DeviceId dstId, PortNumber dstNum, Type type,
+                            Link link) {
+        assertEquals(srcId, link.src().deviceId());
+        assertEquals(srcNum, link.src().port());
+        assertEquals(dstId, link.dst().deviceId());
+        assertEquals(dstNum, link.dst().port());
+        assertEquals(type, link.type());
+    }
+
+    private static void assertLink(LinkKey key, Type type, Link link) {
+        assertLink(key.src().deviceId(), key.src().port(),
+                   key.dst().deviceId(), key.dst().port(),
+                   type, link);
+    }
+
+    @Test
+    public final void testGetLinkCount() {
+        assertEquals("initialy empty", 0, linkStore.getLinkCount());
+
+        putLink(DID1, P1, DID2, P2, DIRECT);
+        putLink(DID2, P2, DID1, P1, DIRECT);
+        putLink(DID1, P1, DID2, P2, DIRECT);
+
+        assertEquals("expecting 2 unique link", 2, linkStore.getLinkCount());
+    }
+
+    @Test
+    public final void testGetLinks() {
+        assertEquals("initialy empty", 0,
+                Iterables.size(linkStore.getLinks()));
+
+        LinkKey linkId1 = new LinkKey(new ConnectPoint(DID1, P1), new ConnectPoint(DID2, P2));
+        LinkKey linkId2 = new LinkKey(new ConnectPoint(DID2, P2), new ConnectPoint(DID1, P1));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId1, DIRECT);
+
+        assertEquals("expecting 2 unique link", 2,
+                Iterables.size(linkStore.getLinks()));
+
+        Map<LinkKey, Link> links = new HashMap<>();
+        for (Link link : linkStore.getLinks()) {
+            links.put(new LinkKey(link.src(), link.dst()), link);
+        }
+
+        assertLink(linkId1, DIRECT, links.get(linkId1));
+        assertLink(linkId2, DIRECT, links.get(linkId2));
+    }
+
+    @Test
+    public final void testGetDeviceEgressLinks() {
+        LinkKey linkId1 = new LinkKey(new ConnectPoint(DID1, P1), new ConnectPoint(DID2, P2));
+        LinkKey linkId2 = new LinkKey(new ConnectPoint(DID2, P2), new ConnectPoint(DID1, P1));
+        LinkKey linkId3 = new LinkKey(new ConnectPoint(DID1, P2), new ConnectPoint(DID2, P3));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId3, DIRECT);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        Set<Link> links1 = linkStore.getDeviceEgressLinks(DID1);
+        assertEquals(2, links1.size());
+        // check
+
+        Set<Link> links2 = linkStore.getDeviceEgressLinks(DID2);
+        assertEquals(1, links2.size());
+        assertLink(linkId2, DIRECT, links2.iterator().next());
+    }
+
+    @Test
+    public final void testGetDeviceIngressLinks() {
+        LinkKey linkId1 = new LinkKey(new ConnectPoint(DID1, P1), new ConnectPoint(DID2, P2));
+        LinkKey linkId2 = new LinkKey(new ConnectPoint(DID2, P2), new ConnectPoint(DID1, P1));
+        LinkKey linkId3 = new LinkKey(new ConnectPoint(DID1, P2), new ConnectPoint(DID2, P3));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId3, DIRECT);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        Set<Link> links1 = linkStore.getDeviceIngressLinks(DID2);
+        assertEquals(2, links1.size());
+        // check
+
+        Set<Link> links2 = linkStore.getDeviceIngressLinks(DID1);
+        assertEquals(1, links2.size());
+        assertLink(linkId2, DIRECT, links2.iterator().next());
+    }
+
+    @Test
+    public final void testGetLink() {
+        ConnectPoint src = new ConnectPoint(DID1, P1);
+        ConnectPoint dst = new ConnectPoint(DID2, P2);
+        LinkKey linkId1 = new LinkKey(src, dst);
+
+        putLink(linkId1, DIRECT);
+
+        Link link = linkStore.getLink(src, dst);
+        assertLink(linkId1, DIRECT, link);
+
+        assertNull("There shouldn't be reverese link",
+                linkStore.getLink(dst, src));
+    }
+
+    @Test
+    public final void testGetEgressLinks() {
+        final ConnectPoint d1P1 = new ConnectPoint(DID1, P1);
+        final ConnectPoint d2P2 = new ConnectPoint(DID2, P2);
+        LinkKey linkId1 = new LinkKey(d1P1, d2P2);
+        LinkKey linkId2 = new LinkKey(d2P2, d1P1);
+        LinkKey linkId3 = new LinkKey(new ConnectPoint(DID1, P2), new ConnectPoint(DID2, P3));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId3, DIRECT);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        Set<Link> links1 = linkStore.getEgressLinks(d1P1);
+        assertEquals(1, links1.size());
+        assertLink(linkId1, DIRECT, links1.iterator().next());
+
+        Set<Link> links2 = linkStore.getEgressLinks(d2P2);
+        assertEquals(1, links2.size());
+        assertLink(linkId2, DIRECT, links2.iterator().next());
+    }
+
+    @Test
+    public final void testGetIngressLinks() {
+        final ConnectPoint d1P1 = new ConnectPoint(DID1, P1);
+        final ConnectPoint d2P2 = new ConnectPoint(DID2, P2);
+        LinkKey linkId1 = new LinkKey(d1P1, d2P2);
+        LinkKey linkId2 = new LinkKey(d2P2, d1P1);
+        LinkKey linkId3 = new LinkKey(new ConnectPoint(DID1, P2), new ConnectPoint(DID2, P3));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId3, DIRECT);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        Set<Link> links1 = linkStore.getIngressLinks(d2P2);
+        assertEquals(1, links1.size());
+        assertLink(linkId1, DIRECT, links1.iterator().next());
+
+        Set<Link> links2 = linkStore.getIngressLinks(d1P1);
+        assertEquals(1, links2.size());
+        assertLink(linkId2, DIRECT, links2.iterator().next());
+    }
+
+    @Test
+    public final void testCreateOrUpdateLink() {
+        ConnectPoint src = new ConnectPoint(DID1, P1);
+        ConnectPoint dst = new ConnectPoint(DID2, P2);
+
+        // add link
+        LinkEvent event = linkStore.createOrUpdateLink(PID,
+                    new DefaultLinkDescription(src, dst, INDIRECT));
+
+        assertLink(DID1, P1, DID2, P2, INDIRECT, event.subject());
+        assertEquals(LINK_ADDED, event.type());
+
+        // update link type
+        LinkEvent event2 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT));
+
+        assertLink(DID1, P1, DID2, P2, DIRECT, event2.subject());
+        assertEquals(LINK_UPDATED, event2.type());
+
+        // no change
+        LinkEvent event3 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT));
+
+        assertNull("No change event expected", event3);
+    }
+
+    @Test
+    public final void testRemoveLink() {
+        final ConnectPoint d1P1 = new ConnectPoint(DID1, P1);
+        final ConnectPoint d2P2 = new ConnectPoint(DID2, P2);
+        LinkKey linkId1 = new LinkKey(d1P1, d2P2);
+        LinkKey linkId2 = new LinkKey(d2P2, d1P1);
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        LinkEvent event = linkStore.removeLink(d1P1, d2P2);
+        assertEquals(LINK_REMOVED, event.type());
+        LinkEvent event2 = linkStore.removeLink(d1P1, d2P2);
+        assertNull(event2);
+
+        assertLink(linkId2, DIRECT, linkStore.getLink(d2P2, d1P1));
+    }
+
+    @Test
+    public final void testEvents() throws InterruptedException {
+
+        final ConnectPoint d1P1 = new ConnectPoint(DID1, P1);
+        final ConnectPoint d2P2 = new ConnectPoint(DID2, P2);
+        final LinkKey linkId1 = new LinkKey(d1P1, d2P2);
+
+        final CountDownLatch addLatch = new CountDownLatch(1);
+        LinkStoreDelegate checkAdd = new LinkStoreDelegate() {
+            @Override
+            public void notify(LinkEvent event) {
+                assertEquals(LINK_ADDED, event.type());
+                assertLink(linkId1, INDIRECT, event.subject());
+                addLatch.countDown();
+            }
+        };
+        final CountDownLatch updateLatch = new CountDownLatch(1);
+        LinkStoreDelegate checkUpdate = new LinkStoreDelegate() {
+            @Override
+            public void notify(LinkEvent event) {
+                assertEquals(LINK_UPDATED, event.type());
+                assertLink(linkId1, DIRECT, event.subject());
+                updateLatch.countDown();
+            }
+        };
+        final CountDownLatch removeLatch = new CountDownLatch(1);
+        LinkStoreDelegate checkRemove = new LinkStoreDelegate() {
+            @Override
+            public void notify(LinkEvent event) {
+                assertEquals(LINK_REMOVED, event.type());
+                assertLink(linkId1, DIRECT, event.subject());
+                removeLatch.countDown();
+            }
+        };
+
+        linkStore.setDelegate(checkAdd);
+        putLink(linkId1, INDIRECT);
+        assertTrue("Add event fired", addLatch.await(1, TimeUnit.SECONDS));
+
+        linkStore.unsetDelegate(checkAdd);
+        linkStore.setDelegate(checkUpdate);
+        putLink(linkId1, DIRECT);
+        assertTrue("Update event fired", updateLatch.await(1, TimeUnit.SECONDS));
+
+        linkStore.unsetDelegate(checkUpdate);
+        linkStore.setDelegate(checkRemove);
+        linkStore.removeLink(d1P1, d2P2);
+        assertTrue("Remove event fired", removeLatch.await(1, TimeUnit.SECONDS));
+    }
+
+
+    class TestDistributedLinkStore extends DistributedLinkStore {
+        TestDistributedLinkStore(StoreService storeService) {
+            this.storeService = storeService;
+        }
+    }
+}