Implement service for mastership-based remote Java proxies
Change-Id: I36228fcc79b73550fbf5d6cbbc8b24e01d869957
diff --git a/core/api/src/main/java/org/onosproject/mastership/MastershipProxyFactory.java b/core/api/src/main/java/org/onosproject/mastership/MastershipProxyFactory.java
new file mode 100644
index 0000000..3bc180c
--- /dev/null
+++ b/core/api/src/main/java/org/onosproject/mastership/MastershipProxyFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.mastership;
+
+import org.onosproject.net.DeviceId;
+
+/**
+ * Mastership-based proxy factory.
+ * <p>
+ * The mastership-based proxy factory constructs proxy instances for the master node of a given {@link DeviceId}.
+ * When a proxy method is invoked, the method will be called on the master node for the device.
+ */
+public interface MastershipProxyFactory<T> {
+
+ /**
+ * Returns the proxy for the given device.
+ *
+ * @param deviceId the device identifier
+ * @return the proxy for the given device
+ */
+ T getProxyFor(DeviceId deviceId);
+
+}
diff --git a/core/api/src/main/java/org/onosproject/mastership/MastershipProxyService.java b/core/api/src/main/java/org/onosproject/mastership/MastershipProxyService.java
new file mode 100644
index 0000000..75caad5
--- /dev/null
+++ b/core/api/src/main/java/org/onosproject/mastership/MastershipProxyService.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.mastership;
+
+import org.onosproject.store.service.Serializer;
+
+/**
+ * Manages remote mastership-based proxy services and instances.
+ * <p>
+ * This service can be used to make arbitrary method calls on remote objects in ONOS. The objects on which the methods
+ * are called are referred to as <em>services</em> and are made available to the proxy service by
+ * {@link #registerProxyService(Class, Object, Serializer) registering} a service instance. Services must implement
+ * an <em>interface</em> which can be used to construct a Java proxy. Proxy interfaces may define either synchronous
+ * or asynchronous methods. Synchronous proxy methods will be blocked, and asynchronous proxy methods (which must
+ * return {@link java.util.concurrent.CompletableFuture}) will be proxied using asynchronous intra-cluster
+ * communication. To make a remote call on a proxy service, get an instance of the {@link MastershipProxyFactory} for
+ * the service type using {@link #getProxyFactory(Class, Serializer)}. The proxy factory is responsible for constructing
+ * proxy instances for each node in the cluster. Once a proxy instance is constructed, calls on the proxy instance will
+ * be transparently serialized and sent to the master node for the associated device and will be executed on the proxy
+ * service registered by that node.
+ */
+public interface MastershipProxyService {
+
+ /**
+ * Returns a proxy factory for the given type.
+ * <p>
+ * The proxy {@code type} passed to this method must be an interface. The proxy factory can be used to construct
+ * mastership-based proxy instances for different devices.
+ *
+ * @param type the proxy type
+ * @param serializer the proxy serializer
+ * @param <T> the proxy type
+ * @return the proxy factory
+ * @throws IllegalArgumentException if the {@code type} is not an interface
+ */
+ <T> MastershipProxyFactory<T> getProxyFactory(Class<T> type, Serializer serializer);
+
+ /**
+ * Registers a proxy service.
+ * <p>
+ * The proxy {@code type} passed to this method must be an interface. The {@code proxy} should be an implementation
+ * of that interface on which methods will be called when proxy calls from other nodes are received.
+ *
+ * @param type the proxy type
+ * @param proxy the proxy service
+ * @param serializer the proxy serializer
+ * @param <T> the proxy type
+ * @throws IllegalArgumentException if the {@code type} is not an interface
+ */
+ <T> void registerProxyService(Class<? super T> type, T proxy, Serializer serializer);
+
+ /**
+ * Unregisters the proxy service of the given type.
+ * <p>
+ * Once the proxy service has been unregistered, calls to the proxy instance on this node will fail.
+ *
+ * @param type the proxy service type to unregister
+ */
+ void unregisterProxyService(Class<?> type);
+
+}
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/MastershipProxyManager.java b/core/net/src/main/java/org/onosproject/cluster/impl/MastershipProxyManager.java
new file mode 100644
index 0000000..96cbe3a
--- /dev/null
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/MastershipProxyManager.java
@@ -0,0 +1,382 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster.impl;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.OrderedExecutor;
+import org.onlab.util.Tools;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.mastership.MastershipProxyFactory;
+import org.onosproject.mastership.MastershipProxyService;
+import org.onosproject.mastership.MastershipService;
+import org.onosproject.net.DeviceId;
+import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.Serializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.onlab.util.Tools.groupedThreads;
+
+/**
+ * Mastership proxy service implementation.
+ * <p>
+ * This implementation wraps both the proxy service and the generated proxy instance in additional proxies which check
+ * mastership and route calls to the appropriate proxy instances.
+ */
+@Component(immediate = true)
+@Service
+public class MastershipProxyManager extends AbstractProxyManager implements MastershipProxyService {
+
+ private static final Serializer REQUEST_SERIALIZER =
+ Serializer.using(KryoNamespaces.API, MastershipProxyRequest.class);
+ private static final String MESSAGE_PREFIX = "mastership-proxy";
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ClusterCommunicationService clusterCommunicator;
+
+ private final ExecutorService proxyServiceExecutor =
+ Executors.newFixedThreadPool(8, groupedThreads("onos/proxy", "service-executor", log));
+
+ private final Map<Class, ProxyService> services = Maps.newConcurrentMap();
+ private NodeId localNodeId;
+
+ @Activate
+ public void activate() {
+ this.localNodeId = clusterService.getLocalNode().id();
+ log.info("Started");
+ }
+
+ @Deactivate
+ public void deactivate() {
+ proxyServiceExecutor.shutdownNow();
+ log.info("Stopped");
+ }
+
+ @Override
+ public <T> MastershipProxyFactory<T> getProxyFactory(Class<T> type, Serializer serializer) {
+ checkArgument(type.isInterface(), "proxy type must be an interface");
+ return new MastershipProxyManagerFactory<>(type, serializer);
+ }
+
+ @Override
+ public <T> void registerProxyService(Class<? super T> type, T instance, Serializer serializer) {
+ checkArgument(type.isInterface(), "proxy type must be an interface");
+ Executor executor = new OrderedExecutor(proxyServiceExecutor);
+ services.computeIfAbsent(type, t -> new ProxyService(instance, t, MESSAGE_PREFIX,
+ (i, m, o) -> new SyncOperationService(i, m, o, serializer, executor),
+ (i, m, o) -> new AsyncOperationService(i, m, o, serializer)));
+ }
+
+ @Override
+ public void unregisterProxyService(Class<?> type) {
+ ProxyService service = services.remove(type);
+ if (service != null) {
+ service.close();
+ }
+ }
+
+ /**
+ * Internal proxy factory.
+ */
+ private class MastershipProxyManagerFactory<T> implements MastershipProxyFactory<T> {
+ private final Class<T> type;
+ private final Serializer serializer;
+ private final Map<DeviceId, T> proxyInstances = Maps.newConcurrentMap();
+
+ MastershipProxyManagerFactory(Class<T> type, Serializer serializer) {
+ this.type = type;
+ this.serializer = serializer;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public T getProxyFor(DeviceId deviceId) {
+ // Avoid unnecessary locking of computeIfAbsent if possible.
+ T proxyInstance = proxyInstances.get(deviceId);
+ if (proxyInstance != null) {
+ return proxyInstance;
+ }
+ return proxyInstances.computeIfAbsent(deviceId, id -> (T) Proxy.newProxyInstance(
+ type.getClassLoader(),
+ new Class[]{type},
+ new ProxyInvocationHandler(type, MESSAGE_PREFIX,
+ o -> new SyncOperationHandler(o, deviceId, serializer),
+ o -> new AsyncOperationHandler(o, deviceId, serializer))));
+ }
+ }
+
+ /**
+ * Implementation of the operation service which handles synchronous method calls.
+ */
+ private class SyncOperationService
+ extends OperationService
+ implements Function<MastershipProxyRequest, Object> {
+
+ private final Serializer serializer;
+
+ SyncOperationService(
+ Object instance,
+ Method method,
+ Operation operation,
+ Serializer serializer,
+ Executor executor) {
+ super(instance, method, operation);
+ this.serializer = serializer;
+ clusterCommunicator.addSubscriber(
+ operation.subject(), REQUEST_SERIALIZER::decode, this, REQUEST_SERIALIZER::encode, executor);
+ }
+
+ @Override
+ public Object apply(MastershipProxyRequest request) {
+ NodeId master = mastershipService.getMasterFor(request.deviceId());
+ if (master == null) {
+ throw new IllegalStateException("No master found for device " + request.deviceId());
+ } else if (!Objects.equals(master, localNodeId)) {
+ // If the given node has already been visited by this node, that indicates a cycle in the intra-cluster
+ // communication. Reject the request.
+ if (request.visited().contains(localNodeId)) {
+ throw new IllegalStateException("Ambiguous master for device " + request.deviceId());
+ }
+
+ // If this node is being visited for the first time, update the request with the local node ID
+ // to prevent cyclic communication.
+ return clusterCommunicator.sendAndReceive(
+ request.visit(),
+ operation.subject(),
+ REQUEST_SERIALIZER::encode,
+ REQUEST_SERIALIZER::decode,
+ master);
+ } else {
+ // Unwrap the raw arguments and apply the method call to the registered proxy service.
+ return invoke(request.unwrap(serializer::decode));
+ }
+ }
+
+ @Override
+ void close() {
+ clusterCommunicator.removeSubscriber(operation.subject());
+ }
+ }
+
+ /**
+ * Implementation of the operation service which handles asynchronous method calls.
+ */
+ private class AsyncOperationService
+ extends OperationService
+ implements Function<MastershipProxyRequest, CompletableFuture<Object>> {
+
+ private final Serializer serializer;
+
+ AsyncOperationService(Object instance, Method method, Operation operation, Serializer serializer) {
+ super(instance, method, operation);
+ this.serializer = serializer;
+ clusterCommunicator.addSubscriber(
+ operation.subject(), REQUEST_SERIALIZER::decode, this, REQUEST_SERIALIZER::encode);
+ }
+
+ @Override
+ public CompletableFuture<Object> apply(MastershipProxyRequest request) {
+ NodeId master = mastershipService.getMasterFor(request.deviceId());
+ if (master == null) {
+ return Tools.exceptionalFuture(
+ new IllegalStateException("No master found for device " + request.deviceId()));
+ } else if (!Objects.equals(master, localNodeId)) {
+ // If the given node has already been visited by this node, that indicates a cycle in the intra-cluster
+ // communication. Reject the request.
+ if (request.visited().contains(localNodeId)) {
+ return Tools.exceptionalFuture(
+ new IllegalStateException("Ambiguous master for device " + request.deviceId()));
+ }
+
+ // If this node is being visited for the first time, update the request with the local node ID
+ // to prevent cyclic communication.
+ return clusterCommunicator.sendAndReceive(
+ request.visit(),
+ operation.subject(),
+ REQUEST_SERIALIZER::encode,
+ REQUEST_SERIALIZER::decode,
+ master);
+ } else {
+ // Unwrap the raw arguments and apply the method call to the registered proxy service.
+ return invoke(request.unwrap(serializer::decode));
+ }
+ }
+
+ @Override
+ void close() {
+ clusterCommunicator.removeSubscriber(operation.subject());
+ }
+ }
+
+ /**
+ * Handler for synchronous proxy operations which blocks on {@code ClusterCommunicationService} requests.
+ */
+ private class SyncOperationHandler extends OperationHandler {
+ private final DeviceId deviceId;
+ private final Serializer serializer;
+
+ SyncOperationHandler(Operation operation, DeviceId deviceId, Serializer serializer) {
+ super(operation);
+ this.deviceId = deviceId;
+ this.serializer = serializer;
+ }
+
+ @Override
+ public Object apply(Object[] args) {
+ NodeId master = mastershipService.getMasterFor(deviceId);
+ if (master == null) {
+ throw new IllegalStateException("No master found for device " + deviceId);
+ }
+
+ MastershipProxyRequest request = new MastershipProxyRequest(deviceId, serializer.encode(args));
+ try {
+ return clusterCommunicator.sendAndReceive(
+ request, operation.subject(), REQUEST_SERIALIZER::encode, REQUEST_SERIALIZER::decode, master)
+ .join();
+ } catch (CompletionException e) {
+ if (e.getCause() instanceof RuntimeException) {
+ throw (RuntimeException) e.getCause();
+ } else {
+ throw new IllegalStateException(e.getCause());
+ }
+ }
+ }
+ }
+
+ /**
+ * Handler for asynchronous proxy operations which uses async {@code ClusterCommunicationService} requests.
+ */
+ private class AsyncOperationHandler extends OperationHandler {
+ private final DeviceId deviceId;
+ private final Serializer serializer;
+
+ AsyncOperationHandler(Operation operation, DeviceId deviceId, Serializer serializer) {
+ super(operation);
+ this.deviceId = deviceId;
+ this.serializer = serializer;
+ }
+
+ @Override
+ public Object apply(Object[] args) {
+ NodeId master = mastershipService.getMasterFor(deviceId);
+ if (master == null) {
+ return Tools.exceptionalFuture(new IllegalStateException("No master found for device " + deviceId));
+ }
+
+ MastershipProxyRequest request = new MastershipProxyRequest(deviceId, serializer.encode(args));
+ return clusterCommunicator.sendAndReceive(
+ request, operation.subject(), REQUEST_SERIALIZER::encode, REQUEST_SERIALIZER::decode, master);
+ }
+ }
+
+ /**
+ * Internal arguments wrapper that contains the {@link DeviceId} for mastership checks.
+ */
+ private class MastershipProxyRequest {
+ private final DeviceId deviceId;
+ private final byte[] args;
+ private final Set<NodeId> visited;
+
+ MastershipProxyRequest(DeviceId deviceId, byte[] args) {
+ this(deviceId, args, ImmutableSet.of(localNodeId));
+ }
+
+ MastershipProxyRequest(DeviceId deviceId, byte[] args, Set<NodeId> visited) {
+ this.deviceId = deviceId;
+ this.args = args;
+ this.visited = visited;
+ }
+
+ /**
+ * Returns the device identifier.
+ *
+ * @return the device identifier
+ */
+ DeviceId deviceId() {
+ return deviceId;
+ }
+
+ /**
+ * Returns the function arguments.
+ *
+ * @return the function arguments
+ */
+ byte[] args() {
+ return args;
+ }
+
+ /**
+ * Decodes and returns the function arguments.
+ *
+ * @param decoder the arguments decoder
+ * @return the decoded function arguments
+ */
+ Object[] unwrap(Function<byte[], Object[]> decoder) {
+ return decoder.apply(args());
+ }
+
+ /**
+ * Returns the set of nodes visited by this request.
+ *
+ * @return the set of nodes visited by this request
+ */
+ Set<NodeId> visited() {
+ return visited;
+ }
+
+ /**
+ * Adds the local node to the set of visited nodes for this request, returning an updated request instance.
+ *
+ * @return a new request with the local node ID added to the visited set
+ */
+ MastershipProxyRequest visit() {
+ return new MastershipProxyRequest(deviceId, args, ImmutableSet.<NodeId>builder()
+ .addAll(visited)
+ .add(localNodeId)
+ .build());
+ }
+ }
+}
diff --git a/core/net/src/test/java/org/onosproject/cluster/impl/MastershipProxyManagerTest.java b/core/net/src/test/java/org/onosproject/cluster/impl/MastershipProxyManagerTest.java
new file mode 100644
index 0000000..3b62cdb
--- /dev/null
+++ b/core/net/src/test/java/org/onosproject/cluster/impl/MastershipProxyManagerTest.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster.impl;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
+import org.onlab.packet.IpAddress;
+import org.onosproject.cluster.ClusterServiceAdapter;
+import org.onosproject.cluster.ControllerNode;
+import org.onosproject.cluster.DefaultControllerNode;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.mastership.MastershipProxyFactory;
+import org.onosproject.mastership.MastershipServiceAdapter;
+import org.onosproject.net.DeviceId;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.Serializer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Mastership proxy manager test.
+ */
+public class MastershipProxyManagerTest {
+ @Test
+ public void testProxyManager() throws Exception {
+ TestClusterCommunicationServiceFactory clusterCommunicatorFactory =
+ new TestClusterCommunicationServiceFactory();
+
+ NodeId a = NodeId.nodeId("a");
+ NodeId b = NodeId.nodeId("b");
+
+ DeviceId deviceId = DeviceId.deviceId("a");
+
+ Serializer serializer = Serializer.using(KryoNamespaces.BASIC);
+
+ ProxyInterfaceImpl proxyInterface1 = new ProxyInterfaceImpl();
+ MastershipProxyManager proxyManager1 = new MastershipProxyManager();
+ proxyManager1.clusterService = new ClusterServiceAdapter() {
+ @Override
+ public ControllerNode getLocalNode() {
+ return new DefaultControllerNode(a, IpAddress.valueOf(0));
+ }
+ };
+ proxyManager1.clusterCommunicator = clusterCommunicatorFactory.newCommunicationService(a);
+ proxyManager1.mastershipService = new MastershipServiceAdapter() {
+ @Override
+ public NodeId getMasterFor(DeviceId deviceId) {
+ return b;
+ }
+ };
+ proxyManager1.activate();
+ proxyManager1.registerProxyService(ProxyInterface.class, proxyInterface1, serializer);
+
+ ProxyInterfaceImpl proxyInterface2 = new ProxyInterfaceImpl();
+ MastershipProxyManager proxyManager2 = new MastershipProxyManager();
+ proxyManager2.clusterService = new ClusterServiceAdapter() {
+ @Override
+ public ControllerNode getLocalNode() {
+ return new DefaultControllerNode(b, IpAddress.valueOf(0));
+ }
+ };
+ proxyManager2.clusterCommunicator = clusterCommunicatorFactory.newCommunicationService(b);
+ proxyManager2.mastershipService = new MastershipServiceAdapter() {
+ @Override
+ public NodeId getMasterFor(DeviceId deviceId) {
+ return b;
+ }
+ };
+ proxyManager2.activate();
+ proxyManager2.registerProxyService(ProxyInterface.class, proxyInterface2, serializer);
+
+ MastershipProxyFactory<ProxyInterface> proxyFactory1 =
+ proxyManager1.getProxyFactory(ProxyInterface.class, serializer);
+ assertEquals("Hello world!", proxyFactory1.getProxyFor(deviceId).sync("Hello world!"));
+ assertEquals(1, proxyInterface2.syncCalls.get());
+ assertEquals("Hello world!", proxyFactory1.getProxyFor(deviceId).async("Hello world!").join());
+ assertEquals(1, proxyInterface2.asyncCalls.get());
+
+ MastershipProxyFactory<ProxyInterface> proxyFactory2 =
+ proxyManager2.getProxyFactory(ProxyInterface.class, serializer);
+ assertEquals("Hello world!", proxyFactory2.getProxyFor(deviceId).sync("Hello world!"));
+ assertEquals(2, proxyInterface2.syncCalls.get());
+ assertEquals("Hello world!", proxyFactory2.getProxyFor(deviceId).async("Hello world!").join());
+ assertEquals(2, proxyInterface2.asyncCalls.get());
+
+ proxyManager1.deactivate();
+ proxyManager2.deactivate();
+ }
+
+ interface ProxyInterface {
+ String sync(String arg);
+ CompletableFuture<String> async(String arg);
+ }
+
+ class ProxyInterfaceImpl implements ProxyInterface {
+ private final AtomicInteger syncCalls = new AtomicInteger();
+ private final AtomicInteger asyncCalls = new AtomicInteger();
+
+ @Override
+ public String sync(String arg) {
+ syncCalls.incrementAndGet();
+ return arg;
+ }
+
+ @Override
+ public CompletableFuture<String> async(String arg) {
+ asyncCalls.incrementAndGet();
+ return CompletableFuture.completedFuture(arg);
+ }
+ }
+}