Madan Jampani | 3289fbf | 2016-01-13 14:14:27 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2016 Open Networking Laboratory |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package org.onosproject.store.primitives.impl; |
| 17 | |
| 18 | import java.io.ByteArrayInputStream; |
| 19 | import java.io.ByteArrayOutputStream; |
| 20 | import java.io.DataInputStream; |
| 21 | import java.io.DataOutputStream; |
| 22 | import java.io.IOException; |
| 23 | import java.io.InputStream; |
| 24 | import java.net.InetAddress; |
| 25 | import java.net.UnknownHostException; |
| 26 | import java.util.Map; |
| 27 | import java.util.Objects; |
| 28 | import java.util.concurrent.CompletableFuture; |
| 29 | import java.util.concurrent.atomic.AtomicInteger; |
| 30 | import java.util.function.Consumer; |
| 31 | |
| 32 | import org.apache.commons.io.IOUtils; |
| 33 | import org.onlab.packet.IpAddress; |
| 34 | import org.onlab.util.Tools; |
| 35 | import org.onosproject.store.cluster.messaging.Endpoint; |
| 36 | import org.onosproject.store.cluster.messaging.MessagingService; |
| 37 | |
| 38 | import com.google.common.base.MoreObjects; |
| 39 | import com.google.common.base.Throwables; |
| 40 | import com.google.common.collect.Maps; |
| 41 | |
| 42 | import static com.google.common.base.Preconditions.checkNotNull; |
| 43 | import io.atomix.catalyst.transport.Address; |
| 44 | import io.atomix.catalyst.transport.Connection; |
| 45 | import io.atomix.catalyst.transport.MessageHandler; |
| 46 | import io.atomix.catalyst.transport.TransportException; |
| 47 | import io.atomix.catalyst.util.Assert; |
| 48 | import io.atomix.catalyst.util.Listener; |
| 49 | import io.atomix.catalyst.util.Listeners; |
| 50 | import io.atomix.catalyst.util.ReferenceCounted; |
| 51 | import io.atomix.catalyst.util.concurrent.ThreadContext; |
| 52 | |
| 53 | /** |
| 54 | * {@link Connection} implementation for CopycatTransport. |
| 55 | */ |
| 56 | public class CopycatTransportConnection implements Connection { |
| 57 | |
| 58 | private final Listeners<Throwable> exceptionListeners = new Listeners<>(); |
| 59 | private final Listeners<Connection> closeListeners = new Listeners<>(); |
| 60 | |
| 61 | static final byte SUCCESS = 0x03; |
| 62 | static final byte FAILURE = 0x04; |
| 63 | |
| 64 | private final long connectionId; |
Madan Jampani | 71d13e1 | 2016-01-13 17:14:35 -0800 | [diff] [blame] | 65 | private final CopycatTransport.Mode mode; |
Madan Jampani | 3289fbf | 2016-01-13 14:14:27 -0800 | [diff] [blame] | 66 | private final Address remoteAddress; |
| 67 | private final MessagingService messagingService; |
| 68 | private final String outboundMessageSubject; |
| 69 | private final String inboundMessageSubject; |
| 70 | private final ThreadContext context; |
| 71 | private final Map<Class<?>, InternalHandler> handlers = Maps.newConcurrentMap(); |
| 72 | private final AtomicInteger messagesSent = new AtomicInteger(0); |
| 73 | private final AtomicInteger sendFailures = new AtomicInteger(0); |
| 74 | private final AtomicInteger messagesReceived = new AtomicInteger(0); |
| 75 | private final AtomicInteger receiveFailures = new AtomicInteger(0); |
Madan Jampani | 71d13e1 | 2016-01-13 17:14:35 -0800 | [diff] [blame] | 76 | private final Map<Address, Endpoint> endpointLookupCache = Maps.newConcurrentMap(); |
Madan Jampani | 3289fbf | 2016-01-13 14:14:27 -0800 | [diff] [blame] | 77 | |
| 78 | CopycatTransportConnection(long connectionId, |
| 79 | CopycatTransport.Mode mode, |
| 80 | String clusterName, |
| 81 | Address address, |
| 82 | MessagingService messagingService, |
| 83 | ThreadContext context) { |
| 84 | this.connectionId = connectionId; |
| 85 | this.mode = checkNotNull(mode); |
| 86 | this.remoteAddress = checkNotNull(address); |
| 87 | this.messagingService = checkNotNull(messagingService); |
| 88 | if (mode == CopycatTransport.Mode.CLIENT) { |
| 89 | this.outboundMessageSubject = String.format("onos-copycat-%s", clusterName); |
| 90 | this.inboundMessageSubject = String.format("onos-copycat-%s-%d", clusterName, connectionId); |
| 91 | } else { |
| 92 | this.outboundMessageSubject = String.format("onos-copycat-%s-%d", clusterName, connectionId); |
| 93 | this.inboundMessageSubject = String.format("onos-copycat-%s", clusterName); |
| 94 | } |
| 95 | this.context = checkNotNull(context); |
| 96 | } |
| 97 | |
| 98 | public void setBidirectional() { |
| 99 | messagingService.registerHandler(inboundMessageSubject, (sender, payload) -> { |
| 100 | try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(payload))) { |
| 101 | if (input.readLong() != connectionId) { |
| 102 | throw new IllegalStateException("Invalid connection Id"); |
| 103 | } |
| 104 | return handle(IOUtils.toByteArray(input)); |
| 105 | } catch (IOException e) { |
| 106 | Throwables.propagate(e); |
| 107 | return null; |
| 108 | } |
| 109 | }); |
| 110 | } |
| 111 | |
| 112 | @Override |
| 113 | public <T, U> CompletableFuture<U> send(T message) { |
| 114 | ThreadContext context = ThreadContext.currentContextOrThrow(); |
| 115 | CompletableFuture<U> result = new CompletableFuture<>(); |
| 116 | try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { |
| 117 | new DataOutputStream(baos).writeLong(connectionId); |
| 118 | context.serializer().writeObject(message, baos); |
| 119 | if (message instanceof ReferenceCounted) { |
| 120 | ((ReferenceCounted<?>) message).release(); |
| 121 | } |
| 122 | messagingService.sendAndReceive(toEndpoint(remoteAddress), |
| 123 | outboundMessageSubject, |
| 124 | baos.toByteArray(), |
| 125 | context.executor()) |
| 126 | .whenComplete((r, e) -> { |
| 127 | if (e == null) { |
| 128 | messagesSent.incrementAndGet(); |
| 129 | } else { |
| 130 | sendFailures.incrementAndGet(); |
| 131 | } |
| 132 | handleResponse(r, e, result, context); |
| 133 | }); |
| 134 | } catch (Exception e) { |
| 135 | result.completeExceptionally(new TransportException("Failed to send request", e)); |
| 136 | } |
| 137 | return result; |
| 138 | } |
| 139 | |
| 140 | private <T> void handleResponse(byte[] response, |
| 141 | Throwable error, |
| 142 | CompletableFuture<T> future, |
| 143 | ThreadContext context) { |
| 144 | if (error != null) { |
| 145 | context.execute(() -> future.completeExceptionally(error)); |
| 146 | return; |
| 147 | } |
| 148 | checkNotNull(response); |
| 149 | InputStream input = new ByteArrayInputStream(response); |
| 150 | try { |
| 151 | byte status = (byte) input.read(); |
| 152 | if (status == FAILURE) { |
| 153 | Throwable t = context.serializer().readObject(input); |
| 154 | context.execute(() -> future.completeExceptionally(t)); |
| 155 | } else { |
| 156 | context.execute(() -> future.complete(context.serializer().readObject(input))); |
| 157 | } |
| 158 | } catch (IOException e) { |
| 159 | context.execute(() -> future.completeExceptionally(e)); |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | @Override |
| 164 | public <T, U> Connection handler(Class<T> type, MessageHandler<T, U> handler) { |
| 165 | Assert.notNull(type, "type"); |
| 166 | handlers.put(type, new InternalHandler(handler, ThreadContext.currentContextOrThrow())); |
| 167 | return null; |
| 168 | } |
| 169 | |
| 170 | public CompletableFuture<byte[]> handle(byte[] message) { |
| 171 | try { |
| 172 | Object request = context.serializer().readObject(new ByteArrayInputStream(message)); |
| 173 | InternalHandler handler = handlers.get(request.getClass()); |
| 174 | if (handler == null) { |
| 175 | return Tools.exceptionalFuture(new IllegalStateException( |
| 176 | "No handler registered for " + request.getClass())); |
| 177 | } |
| 178 | return handler.handle(request).handle((result, error) -> { |
| 179 | if (error == null) { |
| 180 | messagesReceived.incrementAndGet(); |
| 181 | } else { |
| 182 | receiveFailures.incrementAndGet(); |
| 183 | } |
| 184 | try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { |
| 185 | baos.write(error != null ? FAILURE : SUCCESS); |
| 186 | context.serializer().writeObject(error != null ? error : result, baos); |
| 187 | return baos.toByteArray(); |
| 188 | } catch (IOException e) { |
| 189 | Throwables.propagate(e); |
| 190 | return null; |
| 191 | } |
| 192 | }); |
| 193 | } catch (Exception e) { |
| 194 | return Tools.exceptionalFuture(e); |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | @Override |
| 199 | public Listener<Throwable> exceptionListener(Consumer<Throwable> listener) { |
| 200 | return exceptionListeners.add(listener); |
| 201 | } |
| 202 | |
| 203 | @Override |
| 204 | public Listener<Connection> closeListener(Consumer<Connection> listener) { |
| 205 | return closeListeners.add(listener); |
| 206 | } |
| 207 | |
| 208 | @Override |
| 209 | public CompletableFuture<Void> close() { |
Madan Jampani | 3289fbf | 2016-01-13 14:14:27 -0800 | [diff] [blame] | 210 | closeListeners.forEach(listener -> listener.accept(this)); |
| 211 | if (mode == CopycatTransport.Mode.CLIENT) { |
| 212 | messagingService.unregisterHandler(inboundMessageSubject); |
| 213 | } |
| 214 | return CompletableFuture.completedFuture(null); |
| 215 | } |
| 216 | |
| 217 | @Override |
| 218 | public int hashCode() { |
| 219 | return Objects.hash(connectionId); |
| 220 | } |
| 221 | |
| 222 | @Override |
| 223 | public boolean equals(Object other) { |
| 224 | if (!(other instanceof CopycatTransportConnection)) { |
| 225 | return false; |
| 226 | } |
| 227 | |
| 228 | return connectionId == ((CopycatTransportConnection) other).connectionId; |
| 229 | } |
| 230 | |
| 231 | @Override |
| 232 | public String toString() { |
| 233 | return MoreObjects.toStringHelper(getClass()) |
| 234 | .add("id", connectionId) |
| 235 | .add("sent", messagesSent.get()) |
| 236 | .add("received", messagesReceived.get()) |
| 237 | .add("sendFailures", sendFailures.get()) |
| 238 | .add("receiveFailures", receiveFailures.get()) |
| 239 | .toString(); |
| 240 | } |
| 241 | |
| 242 | private Endpoint toEndpoint(Address address) { |
Madan Jampani | 71d13e1 | 2016-01-13 17:14:35 -0800 | [diff] [blame] | 243 | return endpointLookupCache.computeIfAbsent(address, a -> { |
| 244 | try { |
| 245 | return new Endpoint(IpAddress.valueOf(InetAddress.getByName(a.host())), a.port()); |
| 246 | } catch (UnknownHostException e) { |
| 247 | Throwables.propagate(e); |
| 248 | return null; |
| 249 | } |
| 250 | }); |
Madan Jampani | 3289fbf | 2016-01-13 14:14:27 -0800 | [diff] [blame] | 251 | } |
| 252 | |
| 253 | @SuppressWarnings("rawtypes") |
| 254 | private final class InternalHandler { |
| 255 | |
| 256 | private final MessageHandler handler; |
| 257 | private final ThreadContext context; |
| 258 | |
| 259 | private InternalHandler(MessageHandler handler, ThreadContext context) { |
| 260 | this.handler = handler; |
| 261 | this.context = context; |
| 262 | } |
| 263 | |
| 264 | @SuppressWarnings("unchecked") |
| 265 | public CompletableFuture<Object> handle(Object message) { |
| 266 | CompletableFuture<Object> answer = new CompletableFuture<>(); |
| 267 | context.execute(() -> handler.handle(message).whenComplete((r, e) -> { |
| 268 | if (e != null) { |
| 269 | answer.completeExceptionally((Throwable) e); |
| 270 | } else { |
| 271 | answer.complete(r); |
| 272 | } |
| 273 | })); |
| 274 | return answer; |
| 275 | } |
| 276 | } |
| 277 | } |