Implement service for registering, managing, and operating on remote Java proxies over the cluster communication service

Change-Id: I4576e3554cfad08747eed847b73fe695e219f3b8
diff --git a/core/api/src/main/java/org/onosproject/cluster/ProxyFactory.java b/core/api/src/main/java/org/onosproject/cluster/ProxyFactory.java
new file mode 100644
index 0000000..271dc10
--- /dev/null
+++ b/core/api/src/main/java/org/onosproject/cluster/ProxyFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster;
+
+/**
+ * Constructs proxy instances for nodes in the cluster.
+ * <p>
+ * The proxy factory constructs proxy instances for the master node of a given {@link NodeId}.
+ * When a proxy method is invoked, the method will be called on the provided node.
+ */
+public interface ProxyFactory<T> {
+
+    /**
+     * Returns the proxy for the given peer.
+     * <p>
+     * The proxy instance is a multiton that is cached internally in the factory, so calling this method
+     *
+     * @param nodeId the peer node identifier
+     * @return the proxy for the given peer
+     */
+    T getProxyFor(NodeId nodeId);
+
+}
diff --git a/core/api/src/main/java/org/onosproject/cluster/ProxyService.java b/core/api/src/main/java/org/onosproject/cluster/ProxyService.java
new file mode 100644
index 0000000..bdacd67
--- /dev/null
+++ b/core/api/src/main/java/org/onosproject/cluster/ProxyService.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster;
+
+import org.onosproject.store.service.Serializer;
+
+/**
+ * Manages remote proxy services and instances.
+ * <p>
+ * This service can be used to make arbitrary method calls on remote objects in ONOS. The objects on which the methods
+ * are called are referred to as <em>services</em> and are made available to the proxy service by
+ * {@link #registerProxyService(Class, Object, Serializer) registering} a service instance. Services must implement
+ * an <em>interface</em> which can be used to construct a Java proxy. Proxy interfaces may define either synchronous
+ * or asynchronous methods. Synchronous proxy methods will be blocked, and asynchronous proxy methods (which must
+ * return {@link java.util.concurrent.CompletableFuture}) will be proxied using asynchronous intra-cluster
+ * communication. To make a remote call on a proxy service, get an instance of the {@link ProxyFactory} for the service
+ * type using {@link #getProxyFactory(Class, Serializer)}. The proxy factory is responsible for constructing proxy
+ * instances for each node in the cluster. Once a proxy instance is constructed, calls on the proxy instance will be
+ * transparently serialized and sent to the associated peer and be executed on the proxy service registered by that
+ * peer.
+ */
+public interface ProxyService {
+
+    /**
+     * Returns a new proxy factory for the given type.
+     * <p>
+     * The proxy {@code type} passed to this method must be an interface. The proxy factory can be used to construct
+     * proxy instances for different nodes in the cluster.
+     *
+     * @param type       the proxy type
+     * @param serializer the proxy serializer
+     * @param <T>        the proxy type
+     * @return the proxy factory
+     * @throws IllegalArgumentException if the {@code type} is not an interface
+     */
+    <T> ProxyFactory<T> getProxyFactory(Class<T> type, Serializer serializer);
+
+    /**
+     * Registers a proxy service.
+     * <p>
+     * The proxy {@code type} passed to this method must be an interface. The {@code proxy} should be an implementation
+     * of that interface on which methods will be called when proxy calls from other nodes are received.
+     *
+     * @param type       the proxy type
+     * @param proxy      the proxy service
+     * @param serializer the proxy serializer
+     * @param <T>        the proxy type
+     * @throws IllegalArgumentException if the {@code type} is not an interface
+     */
+    <T> void registerProxyService(Class<? super T> type, T proxy, Serializer serializer);
+
+    /**
+     * Unregisters the proxy service of the given type.
+     * <p>
+     * Once the proxy service has been unregistered, calls to the proxy instance on this node will fail.
+     *
+     * @param type the proxy service type to unregister
+     */
+    void unregisterProxyService(Class<?> type);
+
+}
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/AbstractProxyManager.java b/core/net/src/main/java/org/onosproject/cluster/impl/AbstractProxyManager.java
new file mode 100644
index 0000000..a3de529
--- /dev/null
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/AbstractProxyManager.java
@@ -0,0 +1,267 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster.impl;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Maps;
+import org.onosproject.store.cluster.messaging.MessageSubject;
+
+/**
+ * Implementation of the proxy service.
+ */
+public abstract class AbstractProxyManager {
+
+    /**
+     * Wrapper for a proxy service which handles registration of proxy methods as {@code ClusterCommunicationService}
+     * subscribers.
+     */
+    class ProxyService {
+        private final Map<Method, OperationService> operations = Maps.newConcurrentMap();
+
+        ProxyService(
+            Object instance,
+            Class<?> type,
+            String prefix,
+            OperationServiceFactory syncServiceFactory,
+            OperationServiceFactory asyncServiceFactory) {
+            operations.putAll(getMethodMap(type, prefix).entrySet().stream()
+                .map(entry -> {
+                    if (entry.getValue().type() == Operation.Type.SYNC) {
+                        return Maps.immutableEntry(
+                            entry.getKey(),
+                            syncServiceFactory.create(instance, entry.getKey(), entry.getValue()));
+                    } else {
+                        return Maps.immutableEntry(
+                            entry.getKey(),
+                            asyncServiceFactory.create(instance, entry.getKey(), entry.getValue()));
+                    }
+                }).collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())));
+        }
+
+        /**
+         * Closes the proxy service.
+         */
+        void close() {
+            operations.values().forEach(operation -> operation.close());
+        }
+    }
+
+    /**
+     * Operation service factory.
+     */
+    @FunctionalInterface
+    interface OperationServiceFactory {
+        OperationService create(Object instance, Method method, Operation operation);
+    }
+
+    /**
+     * Wrapper for a single proxy service operation which handles registration of subscribers and invocation
+     * of service instance methods.
+     */
+    abstract class OperationService {
+        protected final Object instance;
+        protected final Method method;
+        protected final Operation operation;
+
+        OperationService(Object instance, Method method, Operation operation) {
+            this.instance = instance;
+            this.method = method;
+            this.operation = operation;
+        }
+
+        /**
+         * Invokes the method with the given arguments.
+         *
+         * @param args the arguments with which to invoke the operation
+         * @param <T>  the operation return type
+         * @return the operation return value
+         */
+        @SuppressWarnings("unchecked")
+        <T> T invoke(Object[] args) {
+            try {
+                return (T) method.invoke(instance, args);
+            } catch (IllegalAccessException | InvocationTargetException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        /**
+         * Closes the operation service.
+         */
+        abstract void close();
+    }
+
+    /**
+     * Proxy invocation handler which routes proxy method calls to the correct node and subscriber via
+     * {@code ClusterCommunicationService}.
+     */
+    class ProxyInvocationHandler implements InvocationHandler {
+        private final Map<Method, OperationHandler> handlers = Maps.newConcurrentMap();
+
+        ProxyInvocationHandler(
+            Class<?> type,
+            String prefix,
+            OperationHandlerFactory syncHandlerFactory,
+            OperationHandlerFactory asyncHandlerFactory) {
+            handlers.putAll(getMethodMap(type, prefix).entrySet().stream()
+                .map(entry -> {
+                    if (entry.getValue().type() == Operation.Type.SYNC) {
+                        return Maps.immutableEntry(entry.getKey(), syncHandlerFactory.create(entry.getValue()));
+                    } else {
+                        return Maps.immutableEntry(entry.getKey(), asyncHandlerFactory.create(entry.getValue()));
+                    }
+                }).collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())));
+        }
+
+        @Override
+        public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+            OperationHandler handler = handlers.get(method);
+            if (handler == null) {
+                throw new IllegalStateException("Unknown proxy operation " + method.getName());
+            }
+            return handler.apply(args);
+        }
+    }
+
+    /**
+     * Operation handler factory.
+     */
+    @FunctionalInterface
+    interface OperationHandlerFactory {
+        OperationHandler create(Operation operation);
+    }
+
+    /**
+     * Invocation handler for an individual proxy operation.
+     */
+    abstract class OperationHandler implements Function<Object[], Object> {
+        protected final Operation operation;
+
+        OperationHandler(Operation operation) {
+            this.operation = operation;
+        }
+    }
+
+    /**
+     * Recursively finds operations defined by the given type and its implemented interfaces.
+     *
+     * @param type   the type for which to find operations
+     * @param prefix the prefix with which to generate message subjects
+     * @return the operations defined by the given type and its parent interfaces
+     */
+    Map<Method, Operation> getMethodMap(Class<?> type, String prefix) {
+        String service = type.getCanonicalName().replace(".", "-");
+        Map<Method, Operation> methods = new HashMap<>();
+        for (Method method : type.getDeclaredMethods()) {
+            String name = method.getName();
+            if (methods.values().stream().anyMatch(op -> op.name.equals(name))) {
+                throw new IllegalArgumentException("Method " + name + " is ambiguous");
+            }
+
+            Class<?> returnType = method.getReturnType();
+            if (CompletableFuture.class.isAssignableFrom(returnType)) {
+                methods.put(method, new Operation(Operation.Type.ASYNC, prefix, service, name, method));
+            } else {
+                methods.put(method, new Operation(Operation.Type.SYNC, prefix, service, name, method));
+            }
+        }
+        for (Class<?> iface : type.getInterfaces()) {
+            methods.putAll(getMethodMap(iface, prefix));
+        }
+        return methods;
+    }
+
+    /**
+     * Simple data class for proxy operation metadata.
+     */
+    static class Operation {
+
+        /**
+         * Operation type.
+         */
+        enum Type {
+            SYNC,
+            ASYNC,
+        }
+
+        private final Type type;
+        private final String service;
+        private final String name;
+        private final Method method;
+        private final MessageSubject subject;
+
+        Operation(Type type, String prefix, String service, String name, Method method) {
+            this.type = type;
+            this.service = service;
+            this.name = name;
+            this.method = method;
+            this.subject = new MessageSubject(String.format("%s-%s-%s", prefix, service, name));
+        }
+
+        /**
+         * Returns the operation type.
+         *
+         * @return the operation type
+         */
+        Type type() {
+            return type;
+        }
+
+        /**
+         * Returns the service name of the service to which this operation belongs.
+         *
+         * @return the service name of the service to which this operation belongs
+         */
+        String service() {
+            return service;
+        }
+
+        /**
+         * Returns the operation name.
+         *
+         * @return the operation name
+         */
+        String name() {
+            return name;
+        }
+
+        /**
+         * Returns the operation method.
+         *
+         * @return the operation method
+         */
+        Method method() {
+            return method;
+        }
+
+        /**
+         * Returns the operation message subject.
+         *
+         * @return the operation message subject
+         */
+        MessageSubject subject() {
+            return subject;
+        }
+    }
+}
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/ProxyManager.java b/core/net/src/main/java/org/onosproject/cluster/impl/ProxyManager.java
new file mode 100644
index 0000000..a25a0e3
--- /dev/null
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/ProxyManager.java
@@ -0,0 +1,231 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster.impl;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.function.Function;
+
+import com.google.common.collect.Maps;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.OrderedExecutor;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.cluster.ProxyFactory;
+import org.onosproject.cluster.ProxyService;
+import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
+import org.onosproject.store.service.Serializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.onlab.util.Tools.groupedThreads;
+
+/**
+ * Implementation of the proxy service.
+ */
+@Component(immediate = true)
+@Service
+public class ProxyManager extends AbstractProxyManager implements ProxyService {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
+    private static final String MESSAGE_PREFIX = "proxy";
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterCommunicationService clusterCommunicator;
+
+    private final ExecutorService proxyServiceExecutor =
+        Executors.newFixedThreadPool(
+            Math.min(Math.max(Runtime.getRuntime().availableProcessors(), 4), 16),
+            groupedThreads("onos/proxy", "service-executor", log));
+
+    private final Map<Class, ProxyService> services = Maps.newConcurrentMap();
+
+    @Activate
+    public void activate() {
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        proxyServiceExecutor.shutdownNow();
+        log.info("Stopped");
+    }
+
+    @Override
+    public <T> ProxyFactory<T> getProxyFactory(Class<T> type, Serializer serializer) {
+        checkArgument(type.isInterface(), "proxy type must be an interface");
+        return new ProxyManagerFactory<>(type, serializer);
+    }
+
+    @Override
+    public <T> void registerProxyService(Class<? super T> type, T instance, Serializer serializer) {
+        checkArgument(type.isInterface(), "proxy type must be an interface");
+        Executor executor = new OrderedExecutor(proxyServiceExecutor);
+        services.computeIfAbsent(type, t -> new ProxyService(instance, t, MESSAGE_PREFIX,
+            (i, m, o) -> new SyncOperationService(i, m, o, serializer, executor),
+            (i, m, o) -> new AsyncOperationService(i, m, o, serializer)));
+    }
+
+    @Override
+    public void unregisterProxyService(Class<?> type) {
+        ProxyService service = services.remove(type);
+        if (service != null) {
+            service.close();
+        }
+    }
+
+    /**
+     * Internal proxy factory.
+     */
+    private class ProxyManagerFactory<T> implements ProxyFactory<T> {
+        private final Class<T> type;
+        private final Serializer serializer;
+        private final Map<NodeId, T> proxyInstances = Maps.newConcurrentMap();
+
+        ProxyManagerFactory(Class<T> type, Serializer serializer) {
+            this.type = type;
+            this.serializer = serializer;
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public T getProxyFor(NodeId nodeId) {
+            // Avoid unnecessary locking of computeIfAbsent if possible.
+            T proxyInstance = proxyInstances.get(nodeId);
+            if (proxyInstance != null) {
+                return proxyInstance;
+            }
+            return proxyInstances.computeIfAbsent(nodeId, id -> (T) Proxy.newProxyInstance(
+                type.getClassLoader(),
+                new Class[]{type},
+                new ProxyInvocationHandler(type, MESSAGE_PREFIX,
+                    o -> new SyncOperationHandler(o, nodeId, serializer),
+                    o -> new AsyncOperationHandler(o, nodeId, serializer))));
+        }
+    }
+
+    /**
+     * Implementation of the operation service which handles synchronous method calls.
+     */
+    private class SyncOperationService
+        extends OperationService
+        implements Function<Object[], Object> {
+        SyncOperationService(
+            Object instance,
+            Method method,
+            Operation operation,
+            Serializer serializer,
+            Executor executor) {
+            super(instance, method, operation);
+            clusterCommunicator.addSubscriber(
+                operation.subject(), serializer::decode, this, serializer::encode, executor);
+        }
+
+        @Override
+        public Object apply(Object[] args) {
+            return invoke(args);
+        }
+
+        @Override
+        void close() {
+            clusterCommunicator.removeSubscriber(operation.subject());
+        }
+    }
+
+    /**
+     * Implementation of the operation service which handles asynchronous method calls.
+     */
+    private class AsyncOperationService
+        extends OperationService
+        implements Function<Object[], CompletableFuture<Object>> {
+        AsyncOperationService(Object instance, Method method, Operation operation, Serializer serializer) {
+            super(instance, method, operation);
+            clusterCommunicator.addSubscriber(
+                operation.subject(), serializer::decode, this, serializer::encode);
+        }
+
+        @Override
+        public CompletableFuture<Object> apply(Object[] args) {
+            return invoke(args);
+        }
+
+        @Override
+        void close() {
+            clusterCommunicator.removeSubscriber(operation.subject());
+        }
+    }
+
+    /**
+     * Handler for synchronous proxy operations which blocks on {@code ClusterCommunicationService} requests.
+     */
+    private class SyncOperationHandler extends OperationHandler {
+        private final NodeId nodeId;
+        private final Serializer serializer;
+
+        SyncOperationHandler(Operation operation, NodeId nodeId, Serializer serializer) {
+            super(operation);
+            this.nodeId = nodeId;
+            this.serializer = serializer;
+        }
+
+        @Override
+        public Object apply(Object[] args) {
+            try {
+                return clusterCommunicator.sendAndReceive(
+                    args, operation.subject(), serializer::encode, serializer::decode, nodeId)
+                    .join();
+            } catch (CompletionException e) {
+                if (e.getCause() instanceof RuntimeException) {
+                    throw (RuntimeException) e.getCause();
+                } else {
+                    throw new RuntimeException(e.getCause());
+                }
+            }
+        }
+    }
+
+    /**
+     * Handler for asynchronous proxy operations which uses async {@code ClusterCommunicationService} requests.
+     */
+    private class AsyncOperationHandler extends OperationHandler {
+        private final NodeId nodeId;
+        private final Serializer serializer;
+
+        AsyncOperationHandler(Operation operation, NodeId nodeId, Serializer serializer) {
+            super(operation);
+            this.nodeId = nodeId;
+            this.serializer = serializer;
+        }
+
+        @Override
+        public Object apply(Object[] args) {
+            return clusterCommunicator.sendAndReceive(
+                args, operation.subject(), serializer::encode, serializer::decode, nodeId);
+        }
+    }
+}
diff --git a/core/net/src/test/java/org/onosproject/cluster/impl/ProxyManagerTest.java b/core/net/src/test/java/org/onosproject/cluster/impl/ProxyManagerTest.java
new file mode 100644
index 0000000..0185c0d
--- /dev/null
+++ b/core/net/src/test/java/org/onosproject/cluster/impl/ProxyManagerTest.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster.impl;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.cluster.ProxyFactory;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.Serializer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Proxy manager test.
+ */
+public class ProxyManagerTest {
+    @Test
+    public void testProxyManager() throws Exception {
+        TestClusterCommunicationServiceFactory clusterCommunicatorFactory =
+            new TestClusterCommunicationServiceFactory();
+
+        NodeId a = NodeId.nodeId("a");
+        NodeId b = NodeId.nodeId("b");
+
+        Serializer serializer = Serializer.using(KryoNamespaces.BASIC);
+
+        ProxyInterfaceImpl proxyInterface1 = new ProxyInterfaceImpl();
+        ProxyManager proxyManager1 = new ProxyManager();
+        proxyManager1.clusterCommunicator = clusterCommunicatorFactory.newCommunicationService(a);
+        proxyManager1.activate();
+        proxyManager1.registerProxyService(ProxyInterface.class, proxyInterface1, serializer);
+
+        ProxyInterfaceImpl proxyInterface2 = new ProxyInterfaceImpl();
+        ProxyManager proxyManager2 = new ProxyManager();
+        proxyManager2.clusterCommunicator = clusterCommunicatorFactory.newCommunicationService(b);
+        proxyManager2.activate();
+        proxyManager2.registerProxyService(ProxyInterface.class, proxyInterface2, serializer);
+
+        ProxyFactory<ProxyInterface> proxyFactory1 = proxyManager1.getProxyFactory(ProxyInterface.class, serializer);
+        assertEquals("Hello world!", proxyFactory1.getProxyFor(b).sync("Hello world!"));
+        assertEquals(1, proxyInterface2.syncCalls.get());
+        assertEquals("Hello world!", proxyFactory1.getProxyFor(b).async("Hello world!").join());
+        assertEquals(1, proxyInterface2.asyncCalls.get());
+
+        ProxyFactory<ProxyInterface> proxyFactory2 = proxyManager2.getProxyFactory(ProxyInterface.class, serializer);
+        assertEquals("Hello world!", proxyFactory2.getProxyFor(b).sync("Hello world!"));
+        assertEquals(2, proxyInterface2.syncCalls.get());
+        assertEquals("Hello world!", proxyFactory2.getProxyFor(b).async("Hello world!").join());
+        assertEquals(2, proxyInterface2.asyncCalls.get());
+
+        proxyManager1.deactivate();
+        proxyManager2.deactivate();
+    }
+
+    interface ProxyInterface {
+        String sync(String arg);
+        CompletableFuture<String> async(String arg);
+    }
+
+    class ProxyInterfaceImpl implements ProxyInterface {
+        private final AtomicInteger syncCalls = new AtomicInteger();
+        private final AtomicInteger asyncCalls = new AtomicInteger();
+
+        @Override
+        public String sync(String arg) {
+            syncCalls.incrementAndGet();
+            return arg;
+        }
+
+        @Override
+        public CompletableFuture<String> async(String arg) {
+            asyncCalls.incrementAndGet();
+            return CompletableFuture.completedFuture(arg);
+        }
+    }
+}
diff --git a/core/net/src/test/java/org/onosproject/cluster/impl/TestClusterCommunicationService.java b/core/net/src/test/java/org/onosproject/cluster/impl/TestClusterCommunicationService.java
new file mode 100644
index 0000000..680ddd0
--- /dev/null
+++ b/core/net/src/test/java/org/onosproject/cluster/impl/TestClusterCommunicationService.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster.impl;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import com.google.common.collect.Maps;
+import org.onlab.util.Tools;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
+import org.onosproject.store.cluster.messaging.ClusterMessageHandler;
+import org.onosproject.store.cluster.messaging.MessageSubject;
+import org.onosproject.store.cluster.messaging.MessagingException;
+
+/**
+ * Cluster communication service implementation used for testing.
+ */
+public class TestClusterCommunicationService implements ClusterCommunicationService {
+    private final NodeId localNodeId;
+    private final Map<NodeId, TestClusterCommunicationService> nodes;
+    private final Map<MessageSubject, Function<byte[], CompletableFuture<byte[]>>> subscribers =
+            Maps.newConcurrentMap();
+
+    public TestClusterCommunicationService(NodeId localNodeId, Map<NodeId, TestClusterCommunicationService> nodes) {
+        this.localNodeId = localNodeId;
+        this.nodes = nodes;
+        nodes.put(localNodeId, this);
+    }
+
+    @Override
+    public <M> void broadcast(M message, MessageSubject subject, Function<M, byte[]> encoder) {
+        nodes.forEach((nodeId, node) -> {
+            if (!nodeId.equals(localNodeId)) {
+                node.handle(subject, encoder.apply(message));
+            }
+        });
+    }
+
+    @Override
+    public <M> void broadcastIncludeSelf(M message, MessageSubject subject, Function<M, byte[]> encoder) {
+        nodes.values().forEach(node -> node.handle(subject, encoder.apply(message)));
+    }
+
+    @Override
+    public <M> CompletableFuture<Void> unicast(
+            M message, MessageSubject subject, Function<M, byte[]> encoder, NodeId toNodeId) {
+        TestClusterCommunicationService node = nodes.get(toNodeId);
+        if (node != null) {
+            node.handle(subject, encoder.apply(message));
+        }
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public <M> void multicast(M message, MessageSubject subject, Function<M, byte[]> encoder, Set<NodeId> nodeIds) {
+        nodes.entrySet().stream()
+                .filter(e -> nodeIds.contains(e.getKey()))
+                .forEach(e -> e.getValue().handle(subject, encoder.apply(message)));
+    }
+
+    @Override
+    public <M, R> CompletableFuture<R> sendAndReceive(
+            M message,
+            MessageSubject subject,
+            Function<M, byte[]> encoder,
+            Function<byte[], R> decoder,
+            NodeId toNodeId) {
+        TestClusterCommunicationService node = nodes.get(toNodeId);
+        if (node == null) {
+            return Tools.exceptionalFuture(new MessagingException.NoRemoteHandler());
+        }
+        return node.handle(subject, encoder.apply(message)).thenApply(decoder);
+    }
+
+    private CompletableFuture<byte[]> handle(MessageSubject subject, byte[] message) {
+        Function<byte[], CompletableFuture<byte[]>> subscriber = subscribers.get(subject);
+        if (subscriber != null) {
+            return subscriber.apply(message);
+        }
+        return Tools.exceptionalFuture(new MessagingException.NoRemoteHandler());
+    }
+
+    private boolean isSubscriber(MessageSubject subject) {
+        return subscribers.containsKey(subject);
+    }
+
+    @Override
+    public <M, R> void addSubscriber(
+            MessageSubject subject,
+            Function<byte[], M> decoder,
+            Function<M, R> handler,
+            Function<R, byte[]> encoder,
+            Executor executor) {
+        subscribers.put(subject, message -> {
+            CompletableFuture<byte[]> future = new CompletableFuture<>();
+            executor.execute(() -> {
+                try {
+                    future.complete(encoder.apply(handler.apply(decoder.apply(message))));
+                } catch (Exception e) {
+                    future.completeExceptionally(new MessagingException.RemoteHandlerFailure());
+                }
+            });
+            return future;
+        });
+    }
+
+    @Override
+    public <M, R> void addSubscriber(
+            MessageSubject subject,
+            Function<byte[], M> decoder,
+            Function<M, CompletableFuture<R>> handler,
+            Function<R, byte[]> encoder) {
+        subscribers.put(subject, message -> {
+            CompletableFuture<byte[]> future = new CompletableFuture<>();
+            try {
+                handler.apply(decoder.apply(message)).whenComplete((result, error) -> {
+                    if (error == null) {
+                        future.complete(encoder.apply(result));
+                    } else {
+                        future.completeExceptionally(new MessagingException.RemoteHandlerFailure());
+                    }
+                });
+            } catch (Exception e) {
+                future.completeExceptionally(new MessagingException.RemoteHandlerFailure());
+            }
+            return future;
+        });
+    }
+
+    @Override
+    public <M> void addSubscriber(
+            MessageSubject subject,
+            Function<byte[], M> decoder,
+            Consumer<M> handler,
+            Executor executor) {
+        subscribers.put(subject, message -> {
+            CompletableFuture<byte[]> future = new CompletableFuture<>();
+            executor.execute(() -> {
+                try {
+                    handler.accept(decoder.apply(message));
+                    future.complete(null);
+                } catch (Exception e) {
+                    future.completeExceptionally(new MessagingException.RemoteHandlerFailure());
+                }
+            });
+            return future;
+        });
+    }
+
+    @Override
+    public void removeSubscriber(MessageSubject subject) {
+        subscribers.remove(subject);
+    }
+
+    @Override
+    public void addSubscriber(MessageSubject subject, ClusterMessageHandler subscriber, ExecutorService executor) {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/core/net/src/test/java/org/onosproject/cluster/impl/TestClusterCommunicationServiceFactory.java b/core/net/src/test/java/org/onosproject/cluster/impl/TestClusterCommunicationServiceFactory.java
new file mode 100644
index 0000000..1205506
--- /dev/null
+++ b/core/net/src/test/java/org/onosproject/cluster/impl/TestClusterCommunicationServiceFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster.impl;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
+
+/**
+ * Test cluster communication service factory.
+ */
+public class TestClusterCommunicationServiceFactory {
+    private final Map<NodeId, TestClusterCommunicationService> nodes = Maps.newConcurrentMap();
+
+    /**
+     * Creates a new cluster communication service for the given node.
+     *
+     * @param localNodeId the node for which to create the service
+     * @return the communication service for the given node
+     */
+    public ClusterCommunicationService newCommunicationService(NodeId localNodeId) {
+        return new TestClusterCommunicationService(localNodeId, nodes);
+    }
+}