Setting up monitoring of cluster communcation service
Change-Id: I771b23db6920b26b592abc5d5156e9d77cde4f00
diff --git a/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/ClusterCommunicationManager.java b/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/ClusterCommunicationManager.java
index df4ac5c..ffde400 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/ClusterCommunicationManager.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/ClusterCommunicationManager.java
@@ -31,6 +31,7 @@
import org.onosproject.store.cluster.messaging.Endpoint;
import org.onosproject.store.cluster.messaging.MessageSubject;
import org.onosproject.store.cluster.messaging.MessagingService;
+import org.onosproject.utils.MeteringAgent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,6 +48,7 @@
import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
@Component(immediate = true)
@Service
@@ -55,6 +57,18 @@
private final Logger log = LoggerFactory.getLogger(getClass());
+ private final MeteringAgent subjectMeteringAgent = new MeteringAgent(PRIMITIVE_NAME, SUBJECT_PREFIX, true);
+ private final MeteringAgent endpointMeteringAgent = new MeteringAgent(PRIMITIVE_NAME, ENDPOINT_PREFIX, true);
+
+ private static final String PRIMITIVE_NAME = "clusterCommunication";
+ private static final String SUBJECT_PREFIX = "subject";
+ private static final String ENDPOINT_PREFIX = "endpoint";
+
+ private static final String SERIALIZING = "serialization";
+ private static final String DESERIALIZING = "deserialization";
+ private static final String NODE_PREFIX = "node:";
+ private static final String ROUND_TRIP_SUFFIX = ".rtt";
+
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
private ClusterService clusterService;
@@ -110,7 +124,8 @@
byte[] payload = new ClusterMessage(
localNodeId,
subject,
- encoder.apply(message)).getBytes();
+ timeFunction(encoder, subjectMeteringAgent, SERIALIZING).apply(message)
+ ).getBytes();
return doUnicast(subject, payload, toNodeId);
} catch (Exception e) {
return Tools.exceptionalFuture(e);
@@ -125,7 +140,8 @@
byte[] payload = new ClusterMessage(
localNodeId,
subject,
- encoder.apply(message)).getBytes();
+ timeFunction(encoder, subjectMeteringAgent, SERIALIZING).apply(message))
+ .getBytes();
nodes.forEach(nodeId -> doUnicast(subject, payload, nodeId));
}
@@ -139,8 +155,10 @@
ClusterMessage envelope = new ClusterMessage(
clusterService.getLocalNode().id(),
subject,
- encoder.apply(message));
- return sendAndReceive(subject, envelope.getBytes(), toNodeId).thenApply(decoder);
+ timeFunction(encoder, subjectMeteringAgent, SERIALIZING).
+ apply(message));
+ return sendAndReceive(subject, envelope.getBytes(), toNodeId).
+ thenApply(bytes -> timeFunction(decoder, subjectMeteringAgent, DESERIALIZING).apply(bytes));
} catch (Exception e) {
return Tools.exceptionalFuture(e);
}
@@ -157,7 +175,15 @@
ControllerNode node = clusterService.getNode(toNodeId);
checkArgument(node != null, "Unknown nodeId: %s", toNodeId);
Endpoint nodeEp = new Endpoint(node.ip(), node.tcpPort());
- return messagingService.sendAndReceive(nodeEp, subject.value(), payload);
+ final MeteringAgent.Context epContext = endpointMeteringAgent.
+ startTimer(NODE_PREFIX + toNodeId.toString() + ROUND_TRIP_SUFFIX);
+ final MeteringAgent.Context subjectContext = subjectMeteringAgent.
+ startTimer(subject.toString() + ROUND_TRIP_SUFFIX);
+ return messagingService.sendAndReceive(nodeEp, subject.value(), payload).
+ whenComplete((bytes, throwable) -> {
+ subjectContext.stop(throwable);
+ epContext.stop(throwable);
+ });
}
@Override
@@ -213,6 +239,40 @@
executor);
}
+ /**
+ * Performs the timed function, returning the value it would while timing the operation.
+ *
+ * @param timedFunction the function to be timed
+ * @param meter the metering agent to be used to time the function
+ * @param opName the opname to be used when starting the meter
+ * @param <A> The param type of the function
+ * @param <B> The return type of the function
+ * @return the value returned by the timed function
+ */
+ private <A, B> Function<A, B> timeFunction(Function<A, B> timedFunction,
+ MeteringAgent meter, String opName) {
+ checkNotNull(timedFunction);
+ checkNotNull(meter);
+ checkNotNull(opName);
+ return new Function<A, B>() {
+ @Override
+ public B apply(A a) {
+ final MeteringAgent.Context context = meter.startTimer(opName);
+ B result = null;
+ try {
+ result = timedFunction.apply(a);
+ } catch (Exception e) {
+ context.stop(e);
+ throw new RuntimeException(e);
+ } finally {
+ context.stop(null);
+ return result;
+ }
+ }
+ };
+ }
+
+
private class InternalClusterMessageHandler implements BiFunction<Endpoint, byte[], byte[]> {
private ClusterMessageHandler handler;
@@ -243,7 +303,9 @@
@Override
public CompletableFuture<byte[]> apply(Endpoint sender, byte[] bytes) {
- return handler.apply(decoder.apply(ClusterMessage.fromBytes(bytes).payload())).thenApply(encoder);
+ return handler.apply(timeFunction(decoder, subjectMeteringAgent, DESERIALIZING).
+ apply(ClusterMessage.fromBytes(bytes).payload())).
+ thenApply(m -> timeFunction(encoder, subjectMeteringAgent, SERIALIZING).apply(m));
}
}
@@ -258,7 +320,8 @@
@Override
public void accept(Endpoint sender, byte[] bytes) {
- consumer.accept(decoder.apply(ClusterMessage.fromBytes(bytes).payload()));
+ consumer.accept(timeFunction(decoder, subjectMeteringAgent, DESERIALIZING).
+ apply(ClusterMessage.fromBytes(bytes).payload()));
}
}
}
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncAtomicCounter.java b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncAtomicCounter.java
index 1a38727..6a79292 100644
--- a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncAtomicCounter.java
+++ b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncAtomicCounter.java
@@ -16,6 +16,7 @@
package org.onosproject.store.primitives.impl;
import org.onosproject.store.service.AsyncAtomicCounter;
+import org.onosproject.utils.MeteringAgent;
import java.util.concurrent.CompletableFuture;
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncAtomicValue.java b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncAtomicValue.java
index 7bbd9d3..1d42a97 100644
--- a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncAtomicValue.java
+++ b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncAtomicValue.java
@@ -32,6 +32,7 @@
import com.google.common.base.Throwables;
import com.google.common.collect.Maps;
+import org.onosproject.utils.MeteringAgent;
public class DefaultAsyncAtomicValue<V> implements AsyncAtomicValue<V> {
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncConsistentMap.java b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncConsistentMap.java
index 281fd16..958734c 100644
--- a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncConsistentMap.java
+++ b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncConsistentMap.java
@@ -47,6 +47,7 @@
import org.onosproject.store.service.MapEventListener;
import org.onosproject.store.service.Serializer;
import org.onosproject.store.service.Versioned;
+import org.onosproject.utils.MeteringAgent;
import org.slf4j.Logger;
import com.google.common.cache.CacheBuilder;
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncDistributedSet.java b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncDistributedSet.java
index aaa44aa..fe680aa 100644
--- a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncDistributedSet.java
+++ b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultAsyncDistributedSet.java
@@ -33,6 +33,7 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import org.onosproject.utils.MeteringAgent;
/**
* Implementation of {@link AsyncDistributedSet}.
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultDistributedQueue.java b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultDistributedQueue.java
index 8ea1d72..e4cf65f 100644
--- a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultDistributedQueue.java
+++ b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/DefaultDistributedQueue.java
@@ -22,6 +22,7 @@
import org.onosproject.store.service.DistributedPrimitive;
import org.onosproject.store.service.DistributedQueue;
import org.onosproject.store.service.Serializer;
+import org.onosproject.utils.MeteringAgent;
import java.util.List;
import java.util.Set;
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/MeteredAsyncConsistentMap.java b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/MeteredAsyncConsistentMap.java
index 540d0ad..c397f30 100644
--- a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/MeteredAsyncConsistentMap.java
+++ b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/MeteredAsyncConsistentMap.java
@@ -32,6 +32,7 @@
import com.google.common.base.Throwables;
import com.google.common.collect.Maps;
+import org.onosproject.utils.MeteringAgent;
/**
* {@link AsyncConsistentMap} that meters all its operations.
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/MeteringAgent.java b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/MeteringAgent.java
deleted file mode 100644
index 27a422e..0000000
--- a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/MeteringAgent.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Copyright 2015 Open Networking Laboratory
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.onosproject.store.primitives.impl;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Timer;
-import com.google.common.collect.Maps;
-import org.onlab.metrics.MetricsComponent;
-import org.onlab.metrics.MetricsFeature;
-import org.onlab.metrics.MetricsService;
-import org.onlab.osgi.DefaultServiceDirectory;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Agent that implements usage and performance monitoring via the metrics service.
- */
-public class MeteringAgent {
-
- private Counter exceptionCounter;
- private Counter perObjExceptionCounter;
- private MetricsService metricsService;
- private MetricsComponent metricsComponent;
- private MetricsFeature metricsFeature;
- private final Map<String, Timer> perObjOpTimers = Maps.newConcurrentMap();
- private final Map<String, Timer> perOpTimers = Maps.newConcurrentMap();
- private Timer perPrimitiveTimer;
- private Timer perObjTimer;
- private MetricsFeature wildcard;
- private final boolean activated;
- private Context nullTimer;
-
- /**
- * Constructs a new MeteringAgent for a given distributed primitive.
- * Instantiates the metrics service
- * Initializes all the general metrics for that object
- *
- * @param primitiveName Type of primitive to be metered
- * @param objName Global name of the primitive
- * @param activated boolean flag for whether metering is enabled or not
- */
- public MeteringAgent(String primitiveName, String objName, boolean activated) {
- checkNotNull(objName, "Object name cannot be null");
- this.activated = activated;
- nullTimer = new Context(null, "");
- if (this.activated) {
- this.metricsService = DefaultServiceDirectory.getService(MetricsService.class);
- this.metricsComponent = metricsService.registerComponent(primitiveName);
- this.metricsFeature = metricsComponent.registerFeature(objName);
- this.wildcard = metricsComponent.registerFeature("*");
- this.perObjTimer = metricsService.createTimer(metricsComponent, metricsFeature, "*");
- this.perPrimitiveTimer = metricsService.createTimer(metricsComponent, wildcard, "*");
- this.perObjExceptionCounter = metricsService.createCounter(metricsComponent, metricsFeature, "exceptions");
- this.exceptionCounter = metricsService.createCounter(metricsComponent, wildcard, "exceptions");
- }
- }
-
- /**
- * Initializes a specific timer for a given operation.
- *
- * @param op Specific operation being metered
- * @return timer context
- */
- public Context startTimer(String op) {
- if (!activated) {
- return nullTimer;
- }
- // Check if timer exists, if it doesn't creates it
- final Timer currTimer = perObjOpTimers.computeIfAbsent(op, timer ->
- metricsService.createTimer(metricsComponent, metricsFeature, op));
- perOpTimers.computeIfAbsent(op, timer -> metricsService.createTimer(metricsComponent, wildcard, op));
- // Starts timer
- return new Context(currTimer.time(), op);
- }
-
- /**
- * Timer.Context with a specific operation.
- */
- public class Context {
- private final Timer.Context context;
- private final String operation;
-
- /**
- * Constructs Context.
- *
- * @param context context
- * @param operation operation name
- */
- public Context(Timer.Context context, String operation) {
- this.context = context;
- this.operation = operation;
- }
-
- /**
- * Stops timer given a specific context and updates all related metrics.
- * @param e throwable
- */
- public void stop(Throwable e) {
- if (!activated) {
- return;
- }
- if (e == null) {
- //Stop and updates timer with specific measurements per map, per operation
- final long time = context.stop();
- //updates timer with aggregated measurements per map
- perOpTimers.get(operation).update(time, TimeUnit.NANOSECONDS);
- //updates timer with aggregated measurements per map
- perObjTimer.update(time, TimeUnit.NANOSECONDS);
- //updates timer with aggregated measurements per all Consistent Maps
- perPrimitiveTimer.update(time, TimeUnit.NANOSECONDS);
- } else {
- exceptionCounter.inc();
- perObjExceptionCounter.inc();
- }
- }
- }
-
-}