blob: d49964e8f5b330da13b61e34b14dd88c2ade2da5 [file] [log] [blame]
Thomas Vachuska58de4162015-09-10 16:15:33 -07001/*
Brian O'Connor5ab426f2016-04-09 01:19:45 -07002 * Copyright 2015-present Open Networking Laboratory
Thomas Vachuska58de4162015-09-10 16:15:33 -07003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
Madan Jampaniafeebbd2015-05-19 15:26:01 -070016package org.onosproject.store.cluster.messaging.impl;
17
JunHuy Lam39eb4292015-06-26 17:24:23 +090018import com.google.common.base.Strings;
Aaron Kruglikov1b727382016-02-09 16:17:47 -080019import com.google.common.cache.Cache;
20import com.google.common.cache.CacheBuilder;
21import com.google.common.cache.RemovalListener;
22import com.google.common.cache.RemovalNotification;
23import com.google.common.util.concurrent.MoreExecutors;
Madan Jampania9e70a62016-03-02 16:28:18 -080024
Aaron Kruglikov1b727382016-02-09 16:17:47 -080025import io.netty.bootstrap.Bootstrap;
26import io.netty.bootstrap.ServerBootstrap;
27import io.netty.buffer.PooledByteBufAllocator;
28import io.netty.channel.Channel;
29import io.netty.channel.ChannelFuture;
30import io.netty.channel.ChannelHandler;
31import io.netty.channel.ChannelHandlerContext;
32import io.netty.channel.ChannelInitializer;
33import io.netty.channel.ChannelOption;
34import io.netty.channel.EventLoopGroup;
35import io.netty.channel.ServerChannel;
36import io.netty.channel.SimpleChannelInboundHandler;
Jon Hall9a44d6a2017-03-02 18:14:37 -080037import io.netty.channel.WriteBufferWaterMark;
Aaron Kruglikov1b727382016-02-09 16:17:47 -080038import io.netty.channel.epoll.EpollEventLoopGroup;
39import io.netty.channel.epoll.EpollServerSocketChannel;
40import io.netty.channel.epoll.EpollSocketChannel;
41import io.netty.channel.nio.NioEventLoopGroup;
42import io.netty.channel.socket.SocketChannel;
43import io.netty.channel.socket.nio.NioServerSocketChannel;
44import io.netty.channel.socket.nio.NioSocketChannel;
45import org.apache.commons.pool.KeyedPoolableObjectFactory;
46import org.apache.commons.pool.impl.GenericKeyedObjectPool;
Madan Jampaniafeebbd2015-05-19 15:26:01 -070047import org.apache.felix.scr.annotations.Activate;
48import org.apache.felix.scr.annotations.Component;
49import org.apache.felix.scr.annotations.Deactivate;
50import org.apache.felix.scr.annotations.Reference;
51import org.apache.felix.scr.annotations.ReferenceCardinality;
52import org.apache.felix.scr.annotations.Service;
Aaron Kruglikov1b727382016-02-09 16:17:47 -080053import org.onlab.util.Tools;
Madan Jampaniec1df022015-10-13 21:23:03 -070054import org.onosproject.cluster.ClusterMetadataService;
Madan Jampaniafeebbd2015-05-19 15:26:01 -070055import org.onosproject.cluster.ControllerNode;
Madan Jampani05833872016-07-12 23:01:39 -070056import org.onosproject.core.HybridLogicalClockService;
Madan Jampaniafeebbd2015-05-19 15:26:01 -070057import org.onosproject.store.cluster.messaging.Endpoint;
Madan Jampania9e70a62016-03-02 16:28:18 -080058import org.onosproject.store.cluster.messaging.MessagingException;
Aaron Kruglikov1b727382016-02-09 16:17:47 -080059import org.onosproject.store.cluster.messaging.MessagingService;
Madan Jampania9e70a62016-03-02 16:28:18 -080060import org.onosproject.store.cluster.messaging.impl.InternalMessage.Status;
Madan Jampaniafeebbd2015-05-19 15:26:01 -070061import org.slf4j.Logger;
62import org.slf4j.LoggerFactory;
63
Aaron Kruglikov1b727382016-02-09 16:17:47 -080064import javax.net.ssl.KeyManagerFactory;
65import javax.net.ssl.SSLContext;
66import javax.net.ssl.SSLEngine;
67import javax.net.ssl.TrustManagerFactory;
Madan Jampania9e70a62016-03-02 16:28:18 -080068
Aaron Kruglikov1b727382016-02-09 16:17:47 -080069import java.io.FileInputStream;
70import java.io.IOException;
71import java.security.KeyStore;
72import java.util.Map;
Madan Jampania9e70a62016-03-02 16:28:18 -080073import java.util.Optional;
Aaron Kruglikov1b727382016-02-09 16:17:47 -080074import java.util.concurrent.CompletableFuture;
75import java.util.concurrent.ConcurrentHashMap;
76import java.util.concurrent.Executor;
77import java.util.concurrent.RejectedExecutionException;
78import java.util.concurrent.TimeUnit;
79import java.util.concurrent.TimeoutException;
80import java.util.concurrent.atomic.AtomicBoolean;
81import java.util.concurrent.atomic.AtomicLong;
82import java.util.function.BiConsumer;
83import java.util.function.BiFunction;
84import java.util.function.Consumer;
85
Yuta HIGUCHI90a16892016-07-20 20:36:08 -070086import static org.onlab.util.Tools.groupedThreads;
Heedo Kang4a47a302016-02-29 17:40:23 +090087import static org.onosproject.security.AppGuard.checkPermission;
88import static org.onosproject.security.AppPermission.Type.CLUSTER_WRITE;
89
Madan Jampaniafeebbd2015-05-19 15:26:01 -070090/**
91 * Netty based MessagingService.
92 */
Sho SHIMIZU5c396e32016-08-12 15:19:12 -070093@Component(immediate = true)
Madan Jampaniafeebbd2015-05-19 15:26:01 -070094@Service
Aaron Kruglikov1b727382016-02-09 16:17:47 -080095public class NettyMessagingManager implements MessagingService {
96
Madan Jampanif3ab4242016-07-27 17:21:47 -070097 private static final int REPLY_TIME_OUT_MILLIS = 250;
Aaron Kruglikov1b727382016-02-09 16:17:47 -080098 private static final short MIN_KS_LENGTH = 6;
Madan Jampaniafeebbd2015-05-19 15:26:01 -070099
100 private final Logger log = LoggerFactory.getLogger(getClass());
101
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800102 private static final String REPLY_MESSAGE_TYPE = "NETTY_MESSAGING_REQUEST_REPLY";
103
Madan Jampani05833872016-07-12 23:01:39 -0700104 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
105 protected HybridLogicalClockService clockService;
106
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800107 private Endpoint localEp;
108 private int preamble;
109 private final AtomicBoolean started = new AtomicBoolean(false);
110 private final Map<String, Consumer<InternalMessage>> handlers = new ConcurrentHashMap<>();
111 private final AtomicLong messageIdGenerator = new AtomicLong(0);
112 private final Cache<Long, Callback> callbacks = CacheBuilder.newBuilder()
Madan Jampanif3ab4242016-07-27 17:21:47 -0700113 .expireAfterWrite(REPLY_TIME_OUT_MILLIS, TimeUnit.MILLISECONDS)
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800114 .removalListener(new RemovalListener<Long, Callback>() {
115 @Override
116 public void onRemoval(RemovalNotification<Long, Callback> entry) {
117 if (entry.wasEvicted()) {
118 entry.getValue().completeExceptionally(new TimeoutException("Timedout waiting for reply"));
119 }
120 }
121 })
122 .build();
123
124 private final GenericKeyedObjectPool<Endpoint, Connection> channels
Yuta HIGUCHI90a16892016-07-20 20:36:08 -0700125 = new GenericKeyedObjectPool<>(new OnosCommunicationChannelFactory());
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800126
127 private EventLoopGroup serverGroup;
128 private EventLoopGroup clientGroup;
129 private Class<? extends ServerChannel> serverChannelClass;
130 private Class<? extends Channel> clientChannelClass;
131
132 protected static final boolean TLS_DISABLED = false;
133 protected boolean enableNettyTls = TLS_DISABLED;
134
135 protected String ksLocation;
136 protected String tsLocation;
137 protected char[] ksPwd;
138 protected char[] tsPwd;
JunHuy Lam39eb4292015-06-26 17:24:23 +0900139
Madan Jampaniafeebbd2015-05-19 15:26:01 -0700140 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
Madan Jampaniec1df022015-10-13 21:23:03 -0700141 protected ClusterMetadataService clusterMetadataService;
Madan Jampaniafeebbd2015-05-19 15:26:01 -0700142
143 @Activate
144 public void activate() throws Exception {
Madan Jampaniec1df022015-10-13 21:23:03 -0700145 ControllerNode localNode = clusterMetadataService.getLocalNode();
Jonathan Hartd9df7bd2015-11-10 17:10:25 -0800146 getTlsParameters();
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800147
148 if (started.get()) {
149 log.warn("Already running at local endpoint: {}", localEp);
150 return;
151 }
152 this.preamble = clusterMetadataService.getClusterMetadata().getName().hashCode();
153 this.localEp = new Endpoint(localNode.ip(), localNode.tcpPort());
154 channels.setLifo(true);
155 channels.setTestOnBorrow(true);
156 channels.setTestOnReturn(true);
157 channels.setMinEvictableIdleTimeMillis(60_000L);
158 channels.setTimeBetweenEvictionRunsMillis(30_000L);
159 initEventLoopGroup();
160 startAcceptingConnections();
161 started.set(true);
Madan Jampanif3ab4242016-07-27 17:21:47 -0700162 serverGroup.scheduleWithFixedDelay(callbacks::cleanUp, 0, REPLY_TIME_OUT_MILLIS, TimeUnit.MILLISECONDS);
Madan Jampaniafeebbd2015-05-19 15:26:01 -0700163 log.info("Started");
164 }
165
166 @Deactivate
167 public void deactivate() throws Exception {
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800168 if (started.get()) {
169 channels.close();
170 serverGroup.shutdownGracefully();
171 clientGroup.shutdownGracefully();
172 started.set(false);
173 }
Madan Jampaniafeebbd2015-05-19 15:26:01 -0700174 log.info("Stopped");
175 }
JunHuy Lam39eb4292015-06-26 17:24:23 +0900176
Jonathan Hartd9df7bd2015-11-10 17:10:25 -0800177 private void getTlsParameters() {
JunHuy Lam39eb4292015-06-26 17:24:23 +0900178 String tempString = System.getProperty("enableNettyTLS");
Jonathan Hartd9df7bd2015-11-10 17:10:25 -0800179 enableNettyTls = Strings.isNullOrEmpty(tempString) ? TLS_DISABLED : Boolean.parseBoolean(tempString);
180 log.info("enableNettyTLS = {}", enableNettyTls);
181 if (enableNettyTls) {
JunHuy Lam39eb4292015-06-26 17:24:23 +0900182 ksLocation = System.getProperty("javax.net.ssl.keyStore");
183 if (Strings.isNullOrEmpty(ksLocation)) {
Jonathan Hartd9df7bd2015-11-10 17:10:25 -0800184 enableNettyTls = TLS_DISABLED;
JunHuy Lam39eb4292015-06-26 17:24:23 +0900185 return;
186 }
187 tsLocation = System.getProperty("javax.net.ssl.trustStore");
188 if (Strings.isNullOrEmpty(tsLocation)) {
Jonathan Hartd9df7bd2015-11-10 17:10:25 -0800189 enableNettyTls = TLS_DISABLED;
JunHuy Lam39eb4292015-06-26 17:24:23 +0900190 return;
191 }
192 ksPwd = System.getProperty("javax.net.ssl.keyStorePassword").toCharArray();
193 if (MIN_KS_LENGTH > ksPwd.length) {
Jonathan Hartd9df7bd2015-11-10 17:10:25 -0800194 enableNettyTls = TLS_DISABLED;
JunHuy Lam39eb4292015-06-26 17:24:23 +0900195 return;
196 }
197 tsPwd = System.getProperty("javax.net.ssl.trustStorePassword").toCharArray();
198 if (MIN_KS_LENGTH > tsPwd.length) {
Jonathan Hartd9df7bd2015-11-10 17:10:25 -0800199 enableNettyTls = TLS_DISABLED;
JunHuy Lam39eb4292015-06-26 17:24:23 +0900200 return;
201 }
202 }
203 }
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800204 private void initEventLoopGroup() {
205 // try Epoll first and if that does work, use nio.
206 try {
Yuta HIGUCHI90a16892016-07-20 20:36:08 -0700207 clientGroup = new EpollEventLoopGroup(0, groupedThreads("NettyMessagingEvt", "epollC-%d", log));
208 serverGroup = new EpollEventLoopGroup(0, groupedThreads("NettyMessagingEvt", "epollS-%d", log));
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800209 serverChannelClass = EpollServerSocketChannel.class;
210 clientChannelClass = EpollSocketChannel.class;
211 return;
212 } catch (Throwable e) {
213 log.debug("Failed to initialize native (epoll) transport. "
214 + "Reason: {}. Proceeding with nio.", e.getMessage());
215 }
Yuta HIGUCHI90a16892016-07-20 20:36:08 -0700216 clientGroup = new NioEventLoopGroup(0, groupedThreads("NettyMessagingEvt", "nioC-%d", log));
217 serverGroup = new NioEventLoopGroup(0, groupedThreads("NettyMessagingEvt", "nioS-%d", log));
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800218 serverChannelClass = NioServerSocketChannel.class;
219 clientChannelClass = NioSocketChannel.class;
220 }
221
222 @Override
223 public CompletableFuture<Void> sendAsync(Endpoint ep, String type, byte[] payload) {
Heedo Kang4a47a302016-02-29 17:40:23 +0900224 checkPermission(CLUSTER_WRITE);
Madan Jampanib825aeb2016-04-01 15:18:25 -0700225 InternalMessage message = new InternalMessage(preamble,
Madan Jampani05833872016-07-12 23:01:39 -0700226 clockService.timeNow(),
Madan Jampanib825aeb2016-04-01 15:18:25 -0700227 messageIdGenerator.incrementAndGet(),
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800228 localEp,
229 type,
230 payload);
231 return sendAsync(ep, message);
232 }
233
234 protected CompletableFuture<Void> sendAsync(Endpoint ep, InternalMessage message) {
Heedo Kang4a47a302016-02-29 17:40:23 +0900235 checkPermission(CLUSTER_WRITE);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800236 if (ep.equals(localEp)) {
237 try {
238 dispatchLocally(message);
239 } catch (IOException e) {
240 return Tools.exceptionalFuture(e);
241 }
242 return CompletableFuture.completedFuture(null);
243 }
244
245 CompletableFuture<Void> future = new CompletableFuture<>();
246 try {
247 Connection connection = null;
248 try {
249 connection = channels.borrowObject(ep);
250 connection.send(message, future);
251 } finally {
Yuta HIGUCHIb47c9532016-08-22 09:41:23 -0700252 if (connection != null) {
253 channels.returnObject(ep, connection);
254 }
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800255 }
256 } catch (Exception e) {
257 future.completeExceptionally(e);
258 }
259 return future;
260 }
261
262 @Override
263 public CompletableFuture<byte[]> sendAndReceive(Endpoint ep, String type, byte[] payload) {
Heedo Kang4a47a302016-02-29 17:40:23 +0900264 checkPermission(CLUSTER_WRITE);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800265 return sendAndReceive(ep, type, payload, MoreExecutors.directExecutor());
266 }
267
268 @Override
269 public CompletableFuture<byte[]> sendAndReceive(Endpoint ep, String type, byte[] payload, Executor executor) {
Heedo Kang4a47a302016-02-29 17:40:23 +0900270 checkPermission(CLUSTER_WRITE);
Jordan Halterman041534b2017-03-21 10:37:33 -0700271 CompletableFuture<byte[]> future = new CompletableFuture<>();
272 Callback callback = new Callback(future, executor);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800273 Long messageId = messageIdGenerator.incrementAndGet();
274 callbacks.put(messageId, callback);
Madan Jampani05833872016-07-12 23:01:39 -0700275 InternalMessage message = new InternalMessage(preamble,
276 clockService.timeNow(),
277 messageId,
278 localEp,
279 type,
280 payload);
Jordan Halterman041534b2017-03-21 10:37:33 -0700281
282 sendAsync(ep, message).whenComplete((response, error) -> {
283 if (error != null) {
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800284 callbacks.invalidate(messageId);
Jordan Halterman041534b2017-03-21 10:37:33 -0700285 callback.completeExceptionally(error);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800286 }
Jordan Halterman041534b2017-03-21 10:37:33 -0700287 });
288 return future;
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800289 }
290
291 @Override
292 public void registerHandler(String type, BiConsumer<Endpoint, byte[]> handler, Executor executor) {
Heedo Kang4a47a302016-02-29 17:40:23 +0900293 checkPermission(CLUSTER_WRITE);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800294 handlers.put(type, message -> executor.execute(() -> handler.accept(message.sender(), message.payload())));
295 }
296
297 @Override
298 public void registerHandler(String type, BiFunction<Endpoint, byte[], byte[]> handler, Executor executor) {
Heedo Kang4a47a302016-02-29 17:40:23 +0900299 checkPermission(CLUSTER_WRITE);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800300 handlers.put(type, message -> executor.execute(() -> {
Madan Jampania9e70a62016-03-02 16:28:18 -0800301 byte[] responsePayload = null;
302 Status status = Status.OK;
303 try {
304 responsePayload = handler.apply(message.sender(), message.payload());
305 } catch (Exception e) {
306 status = Status.ERROR_HANDLER_EXCEPTION;
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800307 }
Madan Jampania9e70a62016-03-02 16:28:18 -0800308 sendReply(message, status, Optional.ofNullable(responsePayload));
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800309 }));
310 }
311
312 @Override
313 public void registerHandler(String type, BiFunction<Endpoint, byte[], CompletableFuture<byte[]>> handler) {
Heedo Kang4a47a302016-02-29 17:40:23 +0900314 checkPermission(CLUSTER_WRITE);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800315 handlers.put(type, message -> {
316 handler.apply(message.sender(), message.payload()).whenComplete((result, error) -> {
Madan Jampania9e70a62016-03-02 16:28:18 -0800317 Status status = error == null ? Status.OK : Status.ERROR_HANDLER_EXCEPTION;
318 sendReply(message, status, Optional.ofNullable(result));
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800319 });
320 });
321 }
322
323 @Override
324 public void unregisterHandler(String type) {
Heedo Kang4a47a302016-02-29 17:40:23 +0900325 checkPermission(CLUSTER_WRITE);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800326 handlers.remove(type);
327 }
328
329 private void startAcceptingConnections() throws InterruptedException {
330 ServerBootstrap b = new ServerBootstrap();
Jon Hall9a44d6a2017-03-02 18:14:37 -0800331 b.childOption(ChannelOption.WRITE_BUFFER_WATER_MARK,
332 new WriteBufferWaterMark(8 * 1024, 32 * 1024));
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800333 b.option(ChannelOption.SO_RCVBUF, 1048576);
334 b.option(ChannelOption.TCP_NODELAY, true);
Yuta HIGUCHIb47c9532016-08-22 09:41:23 -0700335 b.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800336 b.group(serverGroup, clientGroup);
337 b.channel(serverChannelClass);
338 if (enableNettyTls) {
339 b.childHandler(new SslServerCommunicationChannelInitializer());
340 } else {
341 b.childHandler(new OnosCommunicationChannelInitializer());
342 }
343 b.option(ChannelOption.SO_BACKLOG, 128);
344 b.childOption(ChannelOption.SO_KEEPALIVE, true);
345
346 // Bind and start to accept incoming connections.
347 b.bind(localEp.port()).sync().addListener(future -> {
348 if (future.isSuccess()) {
349 log.info("{} accepting incoming connections on port {}", localEp.host(), localEp.port());
350 } else {
Jon Hall9a44d6a2017-03-02 18:14:37 -0800351 log.warn("{} failed to bind to port {} due to {}", localEp.host(), localEp.port(), future.cause());
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800352 }
353 });
354 }
355
356 private class OnosCommunicationChannelFactory
357 implements KeyedPoolableObjectFactory<Endpoint, Connection> {
358
359 @Override
Jon Hall9a44d6a2017-03-02 18:14:37 -0800360 public void activateObject(Endpoint endpoint, Connection connection)
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800361 throws Exception {
362 }
363
364 @Override
365 public void destroyObject(Endpoint ep, Connection connection) throws Exception {
Jon Hall9a44d6a2017-03-02 18:14:37 -0800366 log.debug("Closing connection {} to {}", connection, ep);
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800367 //Is this the right way to destroy?
368 connection.destroy();
369 }
370
371 @Override
372 public Connection makeObject(Endpoint ep) throws Exception {
373 Bootstrap bootstrap = new Bootstrap();
374 bootstrap.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
Jon Hall9a44d6a2017-03-02 18:14:37 -0800375 bootstrap.option(ChannelOption.WRITE_BUFFER_WATER_MARK,
376 new WriteBufferWaterMark(10 * 32 * 1024, 10 * 64 * 1024));
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800377 bootstrap.option(ChannelOption.SO_SNDBUF, 1048576);
378 bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 1000);
379 bootstrap.group(clientGroup);
380 // TODO: Make this faster:
381 // http://normanmaurer.me/presentations/2014-facebook-eng-netty/slides.html#37.0
382 bootstrap.channel(clientChannelClass);
383 bootstrap.option(ChannelOption.SO_KEEPALIVE, true);
384 if (enableNettyTls) {
385 bootstrap.handler(new SslClientCommunicationChannelInitializer());
386 } else {
387 bootstrap.handler(new OnosCommunicationChannelInitializer());
388 }
389 // Start the client.
390 CompletableFuture<Channel> retFuture = new CompletableFuture<>();
Yuta HIGUCHIc012dda2016-08-17 00:43:46 -0700391 ChannelFuture f = bootstrap.connect(ep.host().toInetAddress(), ep.port());
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800392
393 f.addListener(future -> {
394 if (future.isSuccess()) {
395 retFuture.complete(f.channel());
396 } else {
397 retFuture.completeExceptionally(future.cause());
398 }
399 });
400 log.debug("Established a new connection to {}", ep);
401 return new Connection(retFuture);
402 }
403
404 @Override
405 public void passivateObject(Endpoint ep, Connection connection)
406 throws Exception {
407 }
408
409 @Override
410 public boolean validateObject(Endpoint ep, Connection connection) {
411 return connection.validate();
412 }
413 }
414
415 private class SslServerCommunicationChannelInitializer extends ChannelInitializer<SocketChannel> {
416
417 private final ChannelHandler dispatcher = new InboundMessageDispatcher();
418 private final ChannelHandler encoder = new MessageEncoder(preamble);
419
420 @Override
421 protected void initChannel(SocketChannel channel) throws Exception {
422 TrustManagerFactory tmFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
423 KeyStore ts = KeyStore.getInstance("JKS");
424 ts.load(new FileInputStream(tsLocation), tsPwd);
425 tmFactory.init(ts);
426
427 KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
428 KeyStore ks = KeyStore.getInstance("JKS");
429 ks.load(new FileInputStream(ksLocation), ksPwd);
430 kmf.init(ks, ksPwd);
431
432 SSLContext serverContext = SSLContext.getInstance("TLS");
433 serverContext.init(kmf.getKeyManagers(), tmFactory.getTrustManagers(), null);
434
435 SSLEngine serverSslEngine = serverContext.createSSLEngine();
436
437 serverSslEngine.setNeedClientAuth(true);
438 serverSslEngine.setUseClientMode(false);
439 serverSslEngine.setEnabledProtocols(serverSslEngine.getSupportedProtocols());
440 serverSslEngine.setEnabledCipherSuites(serverSslEngine.getSupportedCipherSuites());
441 serverSslEngine.setEnableSessionCreation(true);
442
443 channel.pipeline().addLast("ssl", new io.netty.handler.ssl.SslHandler(serverSslEngine))
444 .addLast("encoder", encoder)
Madan Jampanib825aeb2016-04-01 15:18:25 -0700445 .addLast("decoder", new MessageDecoder())
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800446 .addLast("handler", dispatcher);
447 }
448 }
449
450 private class SslClientCommunicationChannelInitializer extends ChannelInitializer<SocketChannel> {
451
452 private final ChannelHandler dispatcher = new InboundMessageDispatcher();
453 private final ChannelHandler encoder = new MessageEncoder(preamble);
454
455 @Override
456 protected void initChannel(SocketChannel channel) throws Exception {
457 TrustManagerFactory tmFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
458 KeyStore ts = KeyStore.getInstance("JKS");
459 ts.load(new FileInputStream(tsLocation), tsPwd);
460 tmFactory.init(ts);
461
462 KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
463 KeyStore ks = KeyStore.getInstance("JKS");
464 ks.load(new FileInputStream(ksLocation), ksPwd);
465 kmf.init(ks, ksPwd);
466
467 SSLContext clientContext = SSLContext.getInstance("TLS");
468 clientContext.init(kmf.getKeyManagers(), tmFactory.getTrustManagers(), null);
469
470 SSLEngine clientSslEngine = clientContext.createSSLEngine();
471
472 clientSslEngine.setUseClientMode(true);
473 clientSslEngine.setEnabledProtocols(clientSslEngine.getSupportedProtocols());
474 clientSslEngine.setEnabledCipherSuites(clientSslEngine.getSupportedCipherSuites());
475 clientSslEngine.setEnableSessionCreation(true);
476
477 channel.pipeline().addLast("ssl", new io.netty.handler.ssl.SslHandler(clientSslEngine))
478 .addLast("encoder", encoder)
Madan Jampanib825aeb2016-04-01 15:18:25 -0700479 .addLast("decoder", new MessageDecoder())
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800480 .addLast("handler", dispatcher);
481 }
482 }
483
484 private class OnosCommunicationChannelInitializer extends ChannelInitializer<SocketChannel> {
485
486 private final ChannelHandler dispatcher = new InboundMessageDispatcher();
487 private final ChannelHandler encoder = new MessageEncoder(preamble);
488
489 @Override
490 protected void initChannel(SocketChannel channel) throws Exception {
491 channel.pipeline()
492 .addLast("encoder", encoder)
Madan Jampanib825aeb2016-04-01 15:18:25 -0700493 .addLast("decoder", new MessageDecoder())
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800494 .addLast("handler", dispatcher);
495 }
496 }
497
498 @ChannelHandler.Sharable
Yuta HIGUCHIc012dda2016-08-17 00:43:46 -0700499 private class InboundMessageDispatcher extends SimpleChannelInboundHandler<Object> {
500 // Effectively SimpleChannelInboundHandler<InternalMessage>,
501 // had to specify <Object> to avoid Class Loader not being able to find some classes.
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800502
503 @Override
Yuta HIGUCHIc012dda2016-08-17 00:43:46 -0700504 protected void channelRead0(ChannelHandlerContext ctx, Object rawMessage) throws Exception {
505 InternalMessage message = (InternalMessage) rawMessage;
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800506 try {
507 dispatchLocally(message);
508 } catch (RejectedExecutionException e) {
509 log.warn("Unable to dispatch message due to {}", e.getMessage());
510 }
511 }
512
513 @Override
514 public void exceptionCaught(ChannelHandlerContext context, Throwable cause) {
515 log.error("Exception inside channel handling pipeline.", cause);
516 context.close();
517 }
Yuta HIGUCHIc012dda2016-08-17 00:43:46 -0700518
519 /**
520 * Returns true if the given message should be handled.
521 *
522 * @param msg inbound message
523 * @return true if {@code msg} is {@link InternalMessage} instance.
524 *
525 * @see SimpleChannelInboundHandler#acceptInboundMessage(Object)
526 */
527 @Override
528 public final boolean acceptInboundMessage(Object msg) {
529 return msg instanceof InternalMessage;
530 }
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800531 }
Yuta HIGUCHIc012dda2016-08-17 00:43:46 -0700532
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800533 private void dispatchLocally(InternalMessage message) throws IOException {
Madan Jampanib825aeb2016-04-01 15:18:25 -0700534 if (message.preamble() != preamble) {
535 log.debug("Received {} with invalid preamble from {}", message.type(), message.sender());
536 sendReply(message, Status.PROTOCOL_EXCEPTION, Optional.empty());
537 }
Madan Jampani05833872016-07-12 23:01:39 -0700538 clockService.recordEventTime(message.time());
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800539 String type = message.type();
540 if (REPLY_MESSAGE_TYPE.equals(type)) {
541 try {
542 Callback callback =
543 callbacks.getIfPresent(message.id());
544 if (callback != null) {
Madan Jampania9e70a62016-03-02 16:28:18 -0800545 if (message.status() == Status.OK) {
546 callback.complete(message.payload());
547 } else if (message.status() == Status.ERROR_NO_HANDLER) {
548 callback.completeExceptionally(new MessagingException.NoRemoteHandler());
549 } else if (message.status() == Status.ERROR_HANDLER_EXCEPTION) {
550 callback.completeExceptionally(new MessagingException.RemoteHandlerFailure());
Madan Jampanib825aeb2016-04-01 15:18:25 -0700551 } else if (message.status() == Status.PROTOCOL_EXCEPTION) {
Jonathan Harte255cc42016-09-12 14:50:24 -0700552 callback.completeExceptionally(new MessagingException.ProtocolException());
Madan Jampania9e70a62016-03-02 16:28:18 -0800553 }
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800554 } else {
Madan Jampania9e70a62016-03-02 16:28:18 -0800555 log.debug("Received a reply for message id:[{}]. "
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800556 + " from {}. But was unable to locate the"
557 + " request handle", message.id(), message.sender());
558 }
559 } finally {
560 callbacks.invalidate(message.id());
561 }
562 return;
563 }
564 Consumer<InternalMessage> handler = handlers.get(type);
565 if (handler != null) {
566 handler.accept(message);
567 } else {
Jon Hall9a44d6a2017-03-02 18:14:37 -0800568 log.debug("No handler for message type {} from {}", message.type(), message.sender());
Madan Jampania9e70a62016-03-02 16:28:18 -0800569 sendReply(message, Status.ERROR_NO_HANDLER, Optional.empty());
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800570 }
571 }
572
Madan Jampania9e70a62016-03-02 16:28:18 -0800573 private void sendReply(InternalMessage message, Status status, Optional<byte[]> responsePayload) {
Madan Jampanib825aeb2016-04-01 15:18:25 -0700574 InternalMessage response = new InternalMessage(preamble,
Madan Jampani05833872016-07-12 23:01:39 -0700575 clockService.timeNow(),
Madan Jampanib825aeb2016-04-01 15:18:25 -0700576 message.id(),
Madan Jampania9e70a62016-03-02 16:28:18 -0800577 localEp,
578 REPLY_MESSAGE_TYPE,
579 responsePayload.orElse(new byte[0]),
580 status);
581 sendAsync(message.sender(), response).whenComplete((result, error) -> {
582 if (error != null) {
583 log.debug("Failed to respond", error);
584 }
585 });
586 }
587
Aaron Kruglikov1b727382016-02-09 16:17:47 -0800588 private final class Callback {
589 private final CompletableFuture<byte[]> future;
590 private final Executor executor;
591
592 public Callback(CompletableFuture<byte[]> future, Executor executor) {
593 this.future = future;
594 this.executor = executor;
595 }
596
597 public void complete(byte[] value) {
598 executor.execute(() -> future.complete(value));
599 }
600
601 public void completeExceptionally(Throwable error) {
602 executor.execute(() -> future.completeExceptionally(error));
603 }
604 }
605 private final class Connection {
606 private final CompletableFuture<Channel> internalFuture;
607
608 public Connection(CompletableFuture<Channel> internalFuture) {
609 this.internalFuture = internalFuture;
610 }
611
612 /**
613 * Sends a message out on its channel and associated the message with a
614 * completable future used for signaling.
615 * @param message the message to be sent
616 * @param future a future that is completed normally or exceptionally if
617 * message sending succeeds or fails respectively
618 */
619 public void send(Object message, CompletableFuture<Void> future) {
620 internalFuture.whenComplete((channel, throwable) -> {
621 if (throwable == null) {
622 channel.writeAndFlush(message).addListener(channelFuture -> {
623 if (!channelFuture.isSuccess()) {
624 future.completeExceptionally(channelFuture.cause());
625 } else {
626 future.complete(null);
627 }
628 });
629 } else {
630 future.completeExceptionally(throwable);
631 }
632 });
633 }
634
635 /**
636 * Destroys a channel by closing its channel (if it exists) and
637 * cancelling its future.
638 */
639 public void destroy() {
640 Channel channel = internalFuture.getNow(null);
641 if (channel != null) {
642 channel.close();
643 }
644 internalFuture.cancel(false);
645 }
646
647 /**
648 * Determines whether the connection is valid meaning it is either
649 * complete with and active channel
650 * or it has not yet completed.
651 * @return true if the channel has an active connection or has not
652 * yet completed
653 */
654 public boolean validate() {
655 if (internalFuture.isCompletedExceptionally()) {
656 return false;
657 }
658 Channel channel = internalFuture.getNow(null);
659 return channel == null || channel.isActive();
660 }
661 }
JunHuy Lam39eb4292015-06-26 17:24:23 +0900662}