Added a messaging service implementation on top of IOLoop. Added ability to easily switch between netty and io loop (default is netty)
Change-Id: Id9af0756bf0a542f832f3611b486b2ac680b91e4
diff --git a/core/api/src/main/java/org/onosproject/store/cluster/messaging/ClusterMessage.java b/core/api/src/main/java/org/onosproject/store/cluster/messaging/ClusterMessage.java
index 9f4d867..46560e4 100644
--- a/core/api/src/main/java/org/onosproject/store/cluster/messaging/ClusterMessage.java
+++ b/core/api/src/main/java/org/onosproject/store/cluster/messaging/ClusterMessage.java
@@ -15,7 +15,6 @@
*/
package org.onosproject.store.cluster.messaging;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Objects;
@@ -35,6 +34,7 @@
private final NodeId sender;
private final MessageSubject subject;
private final byte[] payload;
+ private transient byte[] response;
/**
* Creates a cluster message.
@@ -77,13 +77,21 @@
}
/**
- * Sends a response to the sender.
+ * Records the response to be sent to the sender.
*
- * @param data payload response.
- * @throws IOException when I/O exception of some sort has occurred
+ * @param data response payload
*/
- public void respond(byte[] data) throws IOException {
- throw new IllegalStateException("One can only respond to message received from others.");
+ public void respond(byte[] data) {
+ response = data;
+ }
+
+ /**
+ * Returns the response to be sent to the sender.
+ *
+ * @return response bytes
+ */
+ public byte[] response() {
+ return response;
}
@Override
diff --git a/core/api/src/main/java/org/onosproject/store/cluster/messaging/Endpoint.java b/core/api/src/main/java/org/onosproject/store/cluster/messaging/Endpoint.java
new file mode 100644
index 0000000..2ac50df
--- /dev/null
+++ b/core/api/src/main/java/org/onosproject/store/cluster/messaging/Endpoint.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2014-2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.store.cluster.messaging;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Objects;
+
+import org.onlab.packet.IpAddress;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Representation of a TCP/UDP communication end point.
+ */
+public final class Endpoint {
+
+ private final int port;
+ private final IpAddress ip;
+
+ public Endpoint(IpAddress host, int port) {
+ this.ip = checkNotNull(host);
+ this.port = port;
+ }
+
+ public IpAddress host() {
+ return ip;
+ }
+
+ public int port() {
+ return port;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(getClass())
+ .add("ip", ip)
+ .add("port", port)
+ .toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(ip, port);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ Endpoint that = (Endpoint) obj;
+ return Objects.equals(this.port, that.port) &&
+ Objects.equals(this.ip, that.ip);
+ }
+}
diff --git a/core/api/src/main/java/org/onosproject/store/cluster/messaging/MessagingService.java b/core/api/src/main/java/org/onosproject/store/cluster/messaging/MessagingService.java
new file mode 100644
index 0000000..3fe335b
--- /dev/null
+++ b/core/api/src/main/java/org/onosproject/store/cluster/messaging/MessagingService.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2014-2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.store.cluster.messaging;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+/**
+ * Interface for low level messaging primitives.
+ */
+public interface MessagingService {
+
+ /**
+ * Sends a message asynchronously to the specified communication end point.
+ * The message is specified using the type and payload.
+ * @param ep end point to send the message to.
+ * @param type type of message.
+ * @param payload message payload bytes.
+ * @throws IOException when I/O exception of some sort has occurred
+ */
+ void sendAsync(Endpoint ep, String type, byte[] payload) throws IOException;
+
+ /**
+ * Sends a message synchronously and waits for a response.
+ * @param ep end point to send the message to.
+ * @param type type of message.
+ * @param payload message payload.
+ * @return a response future
+ */
+ CompletableFuture<byte[]> sendAndReceive(Endpoint ep, String type, byte[] payload);
+
+ /**
+ * Registers a new message handler for message type.
+ * @param type message type.
+ * @param handler message handler
+ * @param executor executor to use for running message handler logic.
+ */
+ void registerHandler(String type, Consumer<byte[]> handler, Executor executor);
+
+ /**
+ * Registers a new message handler for message type.
+ * @param type message type.
+ * @param handler message handler
+ * @param executor executor to use for running message handler logic.
+ */
+ void registerHandler(String type, Function<byte[], byte[]> handler, Executor executor);
+
+ /**
+ * Unregister current handler, if one exists for message type.
+ * @param type message type
+ */
+ void unregisterHandler(String type);
+}
diff --git a/core/store/dist/pom.xml b/core/store/dist/pom.xml
index 7384988..ba456ca 100644
--- a/core/store/dist/pom.xml
+++ b/core/store/dist/pom.xml
@@ -53,6 +53,12 @@
<dependency>
<groupId>org.onosproject</groupId>
+ <artifactId>onlab-nio</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.onosproject</groupId>
<artifactId>onlab-misc</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/core/store/dist/src/main/java/org/onosproject/store/cluster/impl/DistributedClusterStore.java b/core/store/dist/src/main/java/org/onosproject/store/cluster/impl/DistributedClusterStore.java
index f458bfe..c81079d 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/cluster/impl/DistributedClusterStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/cluster/impl/DistributedClusterStore.java
@@ -19,14 +19,12 @@
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.hazelcast.util.AddressUtil;
+
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
import org.apache.felix.scr.annotations.Service;
import org.joda.time.DateTime;
-import org.onlab.netty.Endpoint;
-import org.onlab.netty.Message;
-import org.onlab.netty.MessageHandler;
import org.onlab.netty.NettyMessagingService;
import org.onlab.packet.IpAddress;
import org.onlab.util.KryoNamespace;
@@ -38,6 +36,7 @@
import org.onosproject.cluster.DefaultControllerNode;
import org.onosproject.cluster.NodeId;
import org.onosproject.store.AbstractStore;
+import org.onosproject.store.cluster.messaging.Endpoint;
import org.onosproject.store.consistent.impl.DatabaseDefinition;
import org.onosproject.store.consistent.impl.DatabaseDefinitionStore;
import org.onosproject.store.serializers.KryoNamespaces;
@@ -56,6 +55,7 @@
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -108,7 +108,7 @@
private final Map<NodeId, ControllerNode> allNodes = Maps.newConcurrentMap();
private final Map<NodeId, State> nodeStates = Maps.newConcurrentMap();
private final Map<NodeId, DateTime> nodeStateLastUpdatedTimes = Maps.newConcurrentMap();
- private NettyMessagingService messagingService = new NettyMessagingService();
+ private NettyMessagingService messagingService;
private ScheduledExecutorService heartBeatSender = Executors.newSingleThreadScheduledExecutor(
groupedThreads("onos/cluster/membership", "heartbeat-sender"));
private ExecutorService heartBeatMessageHandler = Executors.newSingleThreadExecutor(
@@ -149,7 +149,6 @@
establishSelfIdentity();
messagingService = new NettyMessagingService(HEARTBEAT_FD_PORT);
-
try {
messagingService.activate();
} catch (InterruptedException e) {
@@ -376,10 +375,10 @@
throw new IllegalStateException("Unable to determine local ip");
}
- private class HeartbeatMessageHandler implements MessageHandler {
+ private class HeartbeatMessageHandler implements Consumer<byte[]> {
@Override
- public void handle(Message message) throws IOException {
- HeartbeatMessage hb = SERIALIZER.decode(message.payload());
+ public void accept(byte[] message) {
+ HeartbeatMessage hb = SERIALIZER.decode(message);
failureDetector.report(hb.source().id());
hb.knownPeers().forEach(node -> {
allNodes.put(node.id(), node);
diff --git a/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/ClusterCommunicationManager.java b/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/ClusterCommunicationManager.java
index 042bda2..6f47b48 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/ClusterCommunicationManager.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/ClusterCommunicationManager.java
@@ -21,18 +21,17 @@
import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.apache.felix.scr.annotations.Service;
-import org.onlab.netty.Endpoint;
-import org.onlab.netty.Message;
-import org.onlab.netty.MessageHandler;
-import org.onlab.netty.MessagingService;
import org.onlab.netty.NettyMessagingService;
+import org.onlab.nio.service.IOLoopMessagingService;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.ControllerNode;
import org.onosproject.cluster.NodeId;
import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
import org.onosproject.store.cluster.messaging.ClusterMessage;
import org.onosproject.store.cluster.messaging.ClusterMessageHandler;
+import org.onosproject.store.cluster.messaging.Endpoint;
import org.onosproject.store.cluster.messaging.MessageSubject;
+import org.onosproject.store.cluster.messaging.MessagingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,17 +63,28 @@
// TODO: This probably should not be a OSGi service.
private MessagingService messagingService;
+ private final boolean useNetty = true;
+
@Activate
public void activate() {
ControllerNode localNode = clusterService.getLocalNode();
- NettyMessagingService netty = new NettyMessagingService(localNode.ip(), localNode.tcpPort());
- // FIXME: workaround until it becomes a service.
- try {
- netty.activate();
- } catch (Exception e) {
- log.error("NettyMessagingService#activate", e);
+ if (useNetty) {
+ NettyMessagingService netty = new NettyMessagingService(localNode.ip(), localNode.tcpPort());
+ try {
+ netty.activate();
+ messagingService = netty;
+ } catch (Exception e) {
+ log.error("NettyMessagingService#activate", e);
+ }
+ } else {
+ IOLoopMessagingService ioLoop = new IOLoopMessagingService(localNode.ip(), localNode.tcpPort());
+ try {
+ ioLoop.activate();
+ messagingService = ioLoop;
+ } catch (Exception e) {
+ log.error("IOLoopMessagingService#activate", e);
+ }
}
- messagingService = netty;
log.info("Started on {}:{}", localNode.ip(), localNode.tcpPort());
}
@@ -83,9 +93,13 @@
// TODO: cleanup messageingService if needed.
// FIXME: workaround until it becomes a service.
try {
- ((NettyMessagingService) messagingService).deactivate();
+ if (useNetty) {
+ ((NettyMessagingService) messagingService).deactivate();
+ } else {
+ ((IOLoopMessagingService) messagingService).deactivate();
+ }
} catch (Exception e) {
- log.error("NettyMessagingService#deactivate", e);
+ log.error("MessagingService#deactivate", e);
}
log.info("Stopped");
}
@@ -232,7 +246,9 @@
public void addSubscriber(MessageSubject subject,
ClusterMessageHandler subscriber,
ExecutorService executor) {
- messagingService.registerHandler(subject.value(), new InternalClusterMessageHandler(subscriber), executor);
+ messagingService.registerHandler(subject.value(),
+ new InternalClusterMessageHandler(subscriber),
+ executor);
}
@Override
@@ -240,31 +256,6 @@
messagingService.unregisterHandler(subject.value());
}
- private final class InternalClusterMessageHandler implements MessageHandler {
-
- private final ClusterMessageHandler handler;
-
- public InternalClusterMessageHandler(ClusterMessageHandler handler) {
- this.handler = handler;
- }
-
- @Override
- public void handle(Message message) {
- final ClusterMessage clusterMessage;
- try {
- clusterMessage = ClusterMessage.fromBytes(message.payload());
- } catch (Exception e) {
- log.error("Failed decoding {}", message, e);
- throw e;
- }
- try {
- handler.handle(new InternalClusterMessage(clusterMessage, message));
- } catch (Exception e) {
- log.trace("Failed handling {}", clusterMessage, e);
- throw e;
- }
- }
- }
@Override
public <M, R> void addSubscriber(MessageSubject subject,
@@ -287,7 +278,22 @@
executor);
}
- private class InternalMessageResponder<M, R> implements MessageHandler {
+ private class InternalClusterMessageHandler implements Function<byte[], byte[]> {
+ private ClusterMessageHandler handler;
+
+ public InternalClusterMessageHandler(ClusterMessageHandler handler) {
+ this.handler = handler;
+ }
+
+ @Override
+ public byte[] apply(byte[] bytes) {
+ ClusterMessage message = ClusterMessage.fromBytes(bytes);
+ handler.handle(message);
+ return message.response();
+ }
+ }
+
+ private class InternalMessageResponder<M, R> implements Function<byte[], byte[]> {
private final Function<byte[], M> decoder;
private final Function<R, byte[]> encoder;
private final Function<M, R> handler;
@@ -299,14 +305,15 @@
this.encoder = encoder;
this.handler = handler;
}
+
@Override
- public void handle(Message message) throws IOException {
- R response = handler.apply(decoder.apply(ClusterMessage.fromBytes(message.payload()).payload()));
- message.respond(encoder.apply(response));
+ public byte[] apply(byte[] bytes) {
+ R reply = handler.apply(decoder.apply(ClusterMessage.fromBytes(bytes).payload()));
+ return encoder.apply(reply);
}
}
- private class InternalMessageConsumer<M> implements MessageHandler {
+ private class InternalMessageConsumer<M> implements Consumer<byte[]> {
private final Function<byte[], M> decoder;
private final Consumer<M> consumer;
@@ -314,24 +321,10 @@
this.decoder = decoder;
this.consumer = consumer;
}
- @Override
- public void handle(Message message) throws IOException {
- consumer.accept(decoder.apply(ClusterMessage.fromBytes(message.payload()).payload()));
- }
- }
-
- public static final class InternalClusterMessage extends ClusterMessage {
-
- private final Message rawMessage;
-
- public InternalClusterMessage(ClusterMessage clusterMessage, Message rawMessage) {
- super(clusterMessage.sender(), clusterMessage.subject(), clusterMessage.payload());
- this.rawMessage = rawMessage;
- }
@Override
- public void respond(byte[] response) throws IOException {
- rawMessage.respond(response);
+ public void accept(byte[] bytes) {
+ consumer.accept(decoder.apply(ClusterMessage.fromBytes(bytes).payload()));
}
}
}
diff --git a/core/store/dist/src/main/java/org/onosproject/store/flow/impl/DistributedFlowRuleStore.java b/core/store/dist/src/main/java/org/onosproject/store/flow/impl/DistributedFlowRuleStore.java
index 8ae7d2a..7f8769b 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/flow/impl/DistributedFlowRuleStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/flow/impl/DistributedFlowRuleStore.java
@@ -77,7 +77,6 @@
import org.osgi.service.component.ComponentContext;
import org.slf4j.Logger;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -278,11 +277,7 @@
FlowRule rule = SERIALIZER.decode(message.payload());
log.trace("received get flow entry request for {}", rule);
FlowEntry flowEntry = flowTable.getFlowEntry(rule); //getFlowEntryInternal(rule);
- try {
- message.respond(SERIALIZER.encode(flowEntry));
- } catch (IOException e) {
- log.error("Failed to respond back", e);
- }
+ message.respond(SERIALIZER.encode(flowEntry));
}
}, executor);
@@ -293,11 +288,7 @@
DeviceId deviceId = SERIALIZER.decode(message.payload());
log.trace("Received get flow entries request for {} from {}", deviceId, message.sender());
Set<FlowEntry> flowEntries = flowTable.getFlowEntries(deviceId);
- try {
- message.respond(SERIALIZER.encode(flowEntries));
- } catch (IOException e) {
- log.error("Failed to respond to peer's getFlowEntries request", e);
- }
+ message.respond(SERIALIZER.encode(flowEntries));
}
}, executor);
@@ -308,11 +299,7 @@
FlowEntry rule = SERIALIZER.decode(message.payload());
log.trace("received get flow entry request for {}", rule);
FlowRuleEvent event = removeFlowRuleInternal(rule);
- try {
- message.respond(SERIALIZER.encode(event));
- } catch (IOException e) {
- log.error("Failed to respond back", e);
- }
+ message.respond(SERIALIZER.encode(event));
}
}, executor);
}
@@ -691,11 +678,7 @@
// TODO: we might want to wrap response in envelope
// to distinguish sw programming failure and hand over
// it make sense in the latter case to retry immediately.
- try {
- message.respond(SERIALIZER.encode(allFailed));
- } catch (IOException e) {
- log.error("Failed to respond back", e);
- }
+ message.respond(SERIALIZER.encode(allFailed));
return;
}
diff --git a/core/store/dist/src/main/java/org/onosproject/store/flowext/impl/DefaultFlowRuleExtRouter.java b/core/store/dist/src/main/java/org/onosproject/store/flowext/impl/DefaultFlowRuleExtRouter.java
index b609a3c..b7d69b2 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/flowext/impl/DefaultFlowRuleExtRouter.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/flowext/impl/DefaultFlowRuleExtRouter.java
@@ -51,7 +51,6 @@
import org.onosproject.store.serializers.impl.DistributedStoreSerializers;
import org.slf4j.Logger;
-import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
@@ -143,11 +142,7 @@
@Override
public void run() {
FlowExtCompletedOperation result = Futures.getUnchecked(f);
- try {
- message.respond(SERIALIZER.encode(result));
- } catch (IOException e) {
- log.error("Failed to respond back", e);
- }
+ message.respond(SERIALIZER.encode(result));
}
}, futureListeners);
}
diff --git a/core/store/dist/src/main/java/org/onosproject/store/mastership/impl/ConsistentDeviceMastershipStore.java b/core/store/dist/src/main/java/org/onosproject/store/mastership/impl/ConsistentDeviceMastershipStore.java
index 27a33b3..bc31e08 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/mastership/impl/ConsistentDeviceMastershipStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/mastership/impl/ConsistentDeviceMastershipStore.java
@@ -22,7 +22,6 @@
import static org.slf4j.LoggerFactory.getLogger;
import static com.google.common.base.Preconditions.checkArgument;
-import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -300,11 +299,7 @@
@Override
public void handle(ClusterMessage message) {
DeviceId deviceId = SERIALIZER.decode(message.payload());
- try {
- message.respond(SERIALIZER.encode(getRole(localNodeId, deviceId)));
- } catch (IOException e) {
- log.error("Failed to responsd to role query", e);
- }
+ message.respond(SERIALIZER.encode(getRole(localNodeId, deviceId)));
}
}
@@ -318,11 +313,7 @@
@Override
public void handle(ClusterMessage message) {
DeviceId deviceId = SERIALIZER.decode(message.payload());
- try {
- message.respond(SERIALIZER.encode(relinquishRole(localNodeId, deviceId)));
- } catch (IOException e) {
- log.error("Failed to relinquish role.", e);
- }
+ message.respond(SERIALIZER.encode(relinquishRole(localNodeId, deviceId)));
}
}
@@ -371,4 +362,4 @@
return m.matches();
}
-}
\ No newline at end of file
+}
diff --git a/core/store/dist/src/main/java/org/onosproject/store/statistic/impl/DistributedStatisticStore.java b/core/store/dist/src/main/java/org/onosproject/store/statistic/impl/DistributedStatisticStore.java
index 907631d..cfbcec8 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/statistic/impl/DistributedStatisticStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/statistic/impl/DistributedStatisticStore.java
@@ -43,7 +43,6 @@
import org.onosproject.store.serializers.KryoSerializer;
import org.slf4j.Logger;
-import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
@@ -118,11 +117,7 @@
@Override
public void handle(ClusterMessage message) {
ConnectPoint cp = SERIALIZER.decode(message.payload());
- try {
- message.respond(SERIALIZER.encode(getCurrentStatisticInternal(cp)));
- } catch (IOException e) {
- log.error("Failed to respond back", e);
- }
+ message.respond(SERIALIZER.encode(getCurrentStatisticInternal(cp)));
}
}, messageHandlingExecutor);
@@ -131,11 +126,7 @@
@Override
public void handle(ClusterMessage message) {
ConnectPoint cp = SERIALIZER.decode(message.payload());
- try {
- message.respond(SERIALIZER.encode(getPreviousStatisticInternal(cp)));
- } catch (IOException e) {
- log.error("Failed to respond back", e);
- }
+ message.respond(SERIALIZER.encode(getPreviousStatisticInternal(cp)));
}
}, messageHandlingExecutor);
log.info("Started");