ONOS-6758 Enable TLS by default for intra-cluster communication
Default key store location is config/onos.jks with password changeit
Change-Id: I07cbc09abb22fd8e98fe39a012ce0a65d17d8e39
diff --git a/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/NettyMessagingManager.java b/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/NettyMessagingManager.java
index 6200947..193e81a 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/NettyMessagingManager.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/NettyMessagingManager.java
@@ -15,12 +15,10 @@
*/
package org.onosproject.store.cluster.messaging.impl;
-import com.google.common.base.Strings;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.MoreExecutors;
-
import io.netty.bootstrap.Bootstrap;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.buffer.PooledByteBufAllocator;
@@ -69,13 +67,20 @@
import javax.net.ssl.SSLEngine;
import javax.net.ssl.TrustManagerFactory;
+import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.net.ConnectException;
+import java.security.Key;
import java.security.KeyStore;
+import java.security.MessageDigest;
+import java.security.cert.Certificate;
import java.time.Duration;
+import java.util.Enumeration;
import java.util.Iterator;
import java.util.Map;
import java.util.Optional;
+import java.util.StringJoiner;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@@ -118,6 +123,12 @@
private final ClientConnection localClientConnection = new LocalClientConnection();
private final ServerConnection localServerConnection = new LocalServerConnection(null);
+ //TODO CONFIG_DIR is duplicated from ConfigFileBasedClusterMetadataProvider
+ private static final String CONFIG_DIR = "../config";
+ private static final String KS_FILE_NAME = "onos.jks";
+ private static final File DEFAULT_KS_FILE = new File(CONFIG_DIR, KS_FILE_NAME);
+ private static final String DEFAULT_KS_PASSWORD = "changeit";
+
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected HybridLogicalClockService clockService;
@@ -148,13 +159,12 @@
private Class<? extends Channel> clientChannelClass;
private ScheduledExecutorService timeoutExecutor;
+ protected static final boolean TLS_ENABLED = true;
protected static final boolean TLS_DISABLED = false;
- protected boolean enableNettyTls = TLS_DISABLED;
+ protected boolean enableNettyTls = TLS_ENABLED;
- protected String ksLocation;
- protected String tsLocation;
- protected char[] ksPwd;
- protected char[] tsPwd;
+ protected TrustManagerFactory trustManager;
+ protected KeyManagerFactory keyManager;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ClusterMetadataService clusterMetadataService;
@@ -193,29 +203,76 @@
}
private void getTlsParameters() {
- String tempString = System.getProperty("enableNettyTLS");
- enableNettyTls = Strings.isNullOrEmpty(tempString) ? TLS_DISABLED : Boolean.parseBoolean(tempString);
- log.info("enableNettyTLS = {}", enableNettyTls);
+ // default is TLS enabled unless key stores cannot be loaded
+ enableNettyTls = Boolean.parseBoolean(System.getProperty("enableNettyTLS", Boolean.toString(TLS_ENABLED)));
+
if (enableNettyTls) {
- ksLocation = System.getProperty("javax.net.ssl.keyStore");
- if (Strings.isNullOrEmpty(ksLocation)) {
- enableNettyTls = TLS_DISABLED;
- return;
+ enableNettyTls = loadKeyStores();
+ }
+ }
+
+ private boolean loadKeyStores() {
+ // Maintain a local copy of the trust and key managers in case anything goes wrong
+ TrustManagerFactory tmf;
+ KeyManagerFactory kmf;
+ try {
+ String ksLocation = System.getProperty("javax.net.ssl.keyStore", DEFAULT_KS_FILE.toString());
+ String tsLocation = System.getProperty("javax.net.ssl.trustStore", DEFAULT_KS_FILE.toString());
+ char[] ksPwd = System.getProperty("javax.net.ssl.keyStorePassword", DEFAULT_KS_PASSWORD).toCharArray();
+ char[] tsPwd = System.getProperty("javax.net.ssl.trustStorePassword", DEFAULT_KS_PASSWORD).toCharArray();
+
+ tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ KeyStore ts = KeyStore.getInstance("JKS");
+ ts.load(new FileInputStream(tsLocation), tsPwd);
+ tmf.init(ts);
+
+ kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ KeyStore ks = KeyStore.getInstance("JKS");
+ ks.load(new FileInputStream(ksLocation), ksPwd);
+ kmf.init(ks, ksPwd);
+ if (log.isInfoEnabled()) {
+ logKeyStore(ks, ksLocation, ksPwd);
}
- tsLocation = System.getProperty("javax.net.ssl.trustStore");
- if (Strings.isNullOrEmpty(tsLocation)) {
- enableNettyTls = TLS_DISABLED;
- return;
- }
- ksPwd = System.getProperty("javax.net.ssl.keyStorePassword").toCharArray();
- if (MIN_KS_LENGTH > ksPwd.length) {
- enableNettyTls = TLS_DISABLED;
- return;
- }
- tsPwd = System.getProperty("javax.net.ssl.trustStorePassword").toCharArray();
- if (MIN_KS_LENGTH > tsPwd.length) {
- enableNettyTls = TLS_DISABLED;
- return;
+ } catch (FileNotFoundException e) {
+ log.warn("Disabling TLS for intra-cluster messaging; Could not load cluster key store: {}", e.getMessage());
+ return TLS_DISABLED;
+ } catch (Exception e) {
+ //TODO we might want to catch exceptions more specifically
+ log.error("Error loading key store; disabling TLS for intra-cluster messaging", e);
+ return TLS_DISABLED;
+ }
+ this.trustManager = tmf;
+ this.keyManager = kmf;
+ return TLS_ENABLED;
+ }
+
+ private void logKeyStore(KeyStore ks, String ksLocation, char[] ksPwd) {
+ if (log.isInfoEnabled()) {
+ log.info("Loaded cluster key store from: {}", ksLocation);
+ try {
+ for (Enumeration<String> e = ks.aliases(); e.hasMoreElements();) {
+ String alias = e.nextElement();
+ Key key = ks.getKey(alias, ksPwd);
+ Certificate[] certs = ks.getCertificateChain(alias);
+ log.debug("{} -> {}", alias, certs);
+ final byte[] encodedKey;
+ if (certs != null && certs.length > 0) {
+ encodedKey = certs[0].getEncoded();
+ } else {
+ log.info("Could not find cert chain for {}, using fingerprint of key instead...", alias);
+ encodedKey = key.getEncoded();
+ }
+ // Compute the certificate's fingerprint (use the key if certificate cannot be found)
+ MessageDigest digest = MessageDigest.getInstance("SHA1");
+ digest.update(encodedKey);
+ StringJoiner fingerprint = new StringJoiner(":");
+ for (byte b : digest.digest()) {
+ fingerprint.add(String.format("%02X", b));
+ }
+ log.info("{} -> {}", alias, fingerprint);
+ }
+ } catch (Exception e) {
+ log.warn("Unable to print contents of key store: {}", ksLocation, e);
}
}
}
@@ -449,18 +506,8 @@
@Override
protected void initChannel(SocketChannel channel) throws Exception {
- TrustManagerFactory tmFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
- KeyStore ts = KeyStore.getInstance("JKS");
- ts.load(new FileInputStream(tsLocation), tsPwd);
- tmFactory.init(ts);
-
- KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- KeyStore ks = KeyStore.getInstance("JKS");
- ks.load(new FileInputStream(ksLocation), ksPwd);
- kmf.init(ks, ksPwd);
-
SSLContext serverContext = SSLContext.getInstance("TLS");
- serverContext.init(kmf.getKeyManagers(), tmFactory.getTrustManagers(), null);
+ serverContext.init(keyManager.getKeyManagers(), trustManager.getTrustManagers(), null);
SSLEngine serverSslEngine = serverContext.createSSLEngine();
@@ -486,18 +533,8 @@
@Override
protected void initChannel(SocketChannel channel) throws Exception {
- TrustManagerFactory tmFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
- KeyStore ts = KeyStore.getInstance("JKS");
- ts.load(new FileInputStream(tsLocation), tsPwd);
- tmFactory.init(ts);
-
- KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- KeyStore ks = KeyStore.getInstance("JKS");
- ks.load(new FileInputStream(ksLocation), ksPwd);
- kmf.init(ks, ksPwd);
-
SSLContext clientContext = SSLContext.getInstance("TLS");
- clientContext.init(kmf.getKeyManagers(), tmFactory.getTrustManagers(), null);
+ clientContext.init(keyManager.getKeyManagers(), trustManager.getTrustManagers(), null);
SSLEngine clientSslEngine = clientContext.createSSLEngine();
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/CopycatTransportClient.java b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/CopycatTransportClient.java
index c88242c..afa98cc 100644
--- a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/CopycatTransportClient.java
+++ b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/CopycatTransportClient.java
@@ -75,6 +75,8 @@
if (MessagingException.class.isAssignableFrom(rootCause.getClass())) {
wrappedError = new TransportException(error);
}
+ // TODO ONOS-6788 we might consider demoting this warning during startup when there is
+ // a race between the server registering handlers and the client sending messages
log.warn("Connection to {} failed! Reason: {}", address, wrappedError);
future.completeExceptionally(wrappedError);
} else {
diff --git a/tools/build/envDefaults b/tools/build/envDefaults
index 41a9503..fd282c9 100644
--- a/tools/build/envDefaults
+++ b/tools/build/envDefaults
@@ -51,3 +51,6 @@
export ONOS_GROUP="${ONOS_GROUP:-sdn}" # ONOS group on remote system
export ONOS_PWD="rocks" # ONOS user password on remote system
export ONOS_SCENARIOS=$ONOS_ROOT/tools/test/scenarios
+
+export ONOS_CLUSTER_KEY_FILE="/tmp/onos.jks"
+export ONOS_CLUSTER_KEY_PASSWORD="changeit"
\ No newline at end of file
diff --git a/tools/test/bin/onos-gen-cluster-key b/tools/test/bin/onos-gen-cluster-key
new file mode 100755
index 0000000..168a174
--- /dev/null
+++ b/tools/test/bin/onos-gen-cluster-key
@@ -0,0 +1,23 @@
+#!/bin/bash
+# ------------------------------------------------------------------------
+# This script generates a self-signed certificate and private key pair
+# and stores them in a Java keystore. This keystore can be used as the
+# keystore and trust store for client and server ends of TLS connections
+# for all nodes in the cluster.
+# ------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+[ "$1" = "-f" ] && shift && generate_new_key=true
+
+[ "$generate_new_key" = true ] && rm -f $ONOS_CLUSTER_KEY_FILE
+
+keytool -genkey -keystore $ONOS_CLUSTER_KEY_FILE \
+ -storepass $ONOS_CLUSTER_KEY_PASSWORD \
+ -keyalg RSA \
+ -alias onos \
+ -validity 3600 \
+ -keysize 2048 \
+ -dname CN=onos \
+ -keypass $ONOS_CLUSTER_KEY_PASSWORD
\ No newline at end of file
diff --git a/tools/test/bin/onos-install b/tools/test/bin/onos-install
index 2231667..cc00da0 100755
--- a/tools/test/bin/onos-install
+++ b/tools/test/bin/onos-install
@@ -102,5 +102,8 @@
# Configure the ONOS installation
onos-config $node
+# Upload the shared cluster key if present
+[ -f "$ONOS_CLUSTER_KEY_FILE" ] && onos-push-cluster-key $1
+
# Unless -n option was given, attempt to ignite the ONOS service.
[ -z "$nostart" ] && onos-service $node start || true
\ No newline at end of file
diff --git a/tools/test/bin/onos-push-cluster-key b/tools/test/bin/onos-push-cluster-key
new file mode 100755
index 0000000..c2a77f6
--- /dev/null
+++ b/tools/test/bin/onos-push-cluster-key
@@ -0,0 +1,11 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Pushes the cluster key to the ONOS config directory on a remote ONOS node.
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+remote=$ONOS_USER@${1:-$OCI}
+
+scp -q $ONOS_CLUSTER_KEY_FILE $remote:$ONOS_INSTALL_DIR/config/onos.jks
\ No newline at end of file
diff --git a/tools/test/scenarios/setup.xml b/tools/test/scenarios/setup.xml
index 4f45deb..1d1c36a 100644
--- a/tools/test/scenarios/setup.xml
+++ b/tools/test/scenarios/setup.xml
@@ -30,19 +30,21 @@
</group>
<group name="Install">
+ <step name="Generate-Cluster-Key" exec="onos-gen-cluster-key -f" />
+
<group name="Sequential-Install" if="${ONOS_STC_SEQ_START}">
<sequential var="${OC#}"
starts="Sequential-Install-${#}"
ends="Sequential-Install-${#-1}">
<step name="Sequential-Install-${#}" exec="onos-install ${OC#}"
- requires="Push-Bits-${#},Push-Bits,Cleanup"/>
+ requires="Generate-Cluster-Key,Push-Bits-${#},Push-Bits,Cleanup"/>
</sequential>
</group>
<group name="Parallel-Install" unless="${ONOS_STC_SEQ_START}">
<parallel var="${OC#}">
<step name="Parallel-Install-${#}" exec="onos-install ${OC#}"
- requires="Push-Bits-${#},Push-Bits,Cleanup"/>
+ requires="Generate-Cluster-Key,Push-Bits-${#},Push-Bits,Cleanup"/>
</parallel>
</group>
</group>