ONOS-6758 Enable TLS by default for intra-cluster communication

Default key store location is config/onos.jks with password changeit

Change-Id: I07cbc09abb22fd8e98fe39a012ce0a65d17d8e39
diff --git a/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/NettyMessagingManager.java b/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/NettyMessagingManager.java
index 6200947..193e81a 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/NettyMessagingManager.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/cluster/messaging/impl/NettyMessagingManager.java
@@ -15,12 +15,10 @@
  */
 package org.onosproject.store.cluster.messaging.impl;
 
-import com.google.common.base.Strings;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.MoreExecutors;
-
 import io.netty.bootstrap.Bootstrap;
 import io.netty.bootstrap.ServerBootstrap;
 import io.netty.buffer.PooledByteBufAllocator;
@@ -69,13 +67,20 @@
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.TrustManagerFactory;
 
+import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.net.ConnectException;
+import java.security.Key;
 import java.security.KeyStore;
+import java.security.MessageDigest;
+import java.security.cert.Certificate;
 import java.time.Duration;
+import java.util.Enumeration;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
+import java.util.StringJoiner;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
@@ -118,6 +123,12 @@
     private final ClientConnection localClientConnection = new LocalClientConnection();
     private final ServerConnection localServerConnection = new LocalServerConnection(null);
 
+    //TODO CONFIG_DIR is duplicated from ConfigFileBasedClusterMetadataProvider
+    private static final String CONFIG_DIR = "../config";
+    private static final String KS_FILE_NAME = "onos.jks";
+    private static final File DEFAULT_KS_FILE = new File(CONFIG_DIR, KS_FILE_NAME);
+    private static final String DEFAULT_KS_PASSWORD = "changeit";
+
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected HybridLogicalClockService clockService;
 
@@ -148,13 +159,12 @@
     private Class<? extends Channel> clientChannelClass;
     private ScheduledExecutorService timeoutExecutor;
 
+    protected static final boolean TLS_ENABLED = true;
     protected static final boolean TLS_DISABLED = false;
-    protected boolean enableNettyTls = TLS_DISABLED;
+    protected boolean enableNettyTls = TLS_ENABLED;
 
-    protected String ksLocation;
-    protected String tsLocation;
-    protected char[] ksPwd;
-    protected char[] tsPwd;
+    protected TrustManagerFactory trustManager;
+    protected KeyManagerFactory keyManager;
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected ClusterMetadataService clusterMetadataService;
@@ -193,29 +203,76 @@
     }
 
     private void getTlsParameters() {
-        String tempString = System.getProperty("enableNettyTLS");
-        enableNettyTls = Strings.isNullOrEmpty(tempString) ? TLS_DISABLED : Boolean.parseBoolean(tempString);
-        log.info("enableNettyTLS = {}", enableNettyTls);
+        // default is TLS enabled unless key stores cannot be loaded
+        enableNettyTls = Boolean.parseBoolean(System.getProperty("enableNettyTLS", Boolean.toString(TLS_ENABLED)));
+
         if (enableNettyTls) {
-            ksLocation = System.getProperty("javax.net.ssl.keyStore");
-            if (Strings.isNullOrEmpty(ksLocation)) {
-                enableNettyTls = TLS_DISABLED;
-                return;
+            enableNettyTls = loadKeyStores();
+        }
+    }
+
+    private boolean loadKeyStores() {
+        // Maintain a local copy of the trust and key managers in case anything goes wrong
+        TrustManagerFactory tmf;
+        KeyManagerFactory kmf;
+        try {
+            String ksLocation = System.getProperty("javax.net.ssl.keyStore", DEFAULT_KS_FILE.toString());
+            String tsLocation = System.getProperty("javax.net.ssl.trustStore", DEFAULT_KS_FILE.toString());
+            char[] ksPwd = System.getProperty("javax.net.ssl.keyStorePassword", DEFAULT_KS_PASSWORD).toCharArray();
+            char[] tsPwd = System.getProperty("javax.net.ssl.trustStorePassword", DEFAULT_KS_PASSWORD).toCharArray();
+
+            tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+            KeyStore ts = KeyStore.getInstance("JKS");
+            ts.load(new FileInputStream(tsLocation), tsPwd);
+            tmf.init(ts);
+
+            kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+            KeyStore ks = KeyStore.getInstance("JKS");
+            ks.load(new FileInputStream(ksLocation), ksPwd);
+            kmf.init(ks, ksPwd);
+            if (log.isInfoEnabled()) {
+                logKeyStore(ks, ksLocation, ksPwd);
             }
-            tsLocation = System.getProperty("javax.net.ssl.trustStore");
-            if (Strings.isNullOrEmpty(tsLocation)) {
-                enableNettyTls = TLS_DISABLED;
-                return;
-            }
-            ksPwd = System.getProperty("javax.net.ssl.keyStorePassword").toCharArray();
-            if (MIN_KS_LENGTH > ksPwd.length) {
-                enableNettyTls = TLS_DISABLED;
-                return;
-            }
-            tsPwd = System.getProperty("javax.net.ssl.trustStorePassword").toCharArray();
-            if (MIN_KS_LENGTH > tsPwd.length) {
-                enableNettyTls = TLS_DISABLED;
-                return;
+        } catch (FileNotFoundException e) {
+            log.warn("Disabling TLS for intra-cluster messaging; Could not load cluster key store: {}", e.getMessage());
+            return TLS_DISABLED;
+        } catch (Exception e) {
+            //TODO we might want to catch exceptions more specifically
+            log.error("Error loading key store; disabling TLS for intra-cluster messaging", e);
+            return TLS_DISABLED;
+        }
+        this.trustManager = tmf;
+        this.keyManager = kmf;
+        return TLS_ENABLED;
+    }
+
+    private void logKeyStore(KeyStore ks, String ksLocation, char[] ksPwd) {
+        if (log.isInfoEnabled()) {
+            log.info("Loaded cluster key store from: {}", ksLocation);
+            try {
+                for (Enumeration<String> e = ks.aliases(); e.hasMoreElements();) {
+                    String alias = e.nextElement();
+                    Key key = ks.getKey(alias, ksPwd);
+                    Certificate[] certs = ks.getCertificateChain(alias);
+                    log.debug("{} -> {}", alias, certs);
+                    final byte[] encodedKey;
+                    if (certs != null && certs.length > 0) {
+                        encodedKey = certs[0].getEncoded();
+                    } else {
+                        log.info("Could not find cert chain for {}, using fingerprint of key instead...", alias);
+                        encodedKey = key.getEncoded();
+                    }
+                    // Compute the certificate's fingerprint (use the key if certificate cannot be found)
+                    MessageDigest digest = MessageDigest.getInstance("SHA1");
+                    digest.update(encodedKey);
+                    StringJoiner fingerprint = new StringJoiner(":");
+                    for (byte b : digest.digest()) {
+                        fingerprint.add(String.format("%02X", b));
+                    }
+                    log.info("{} -> {}", alias, fingerprint);
+                }
+            } catch (Exception e) {
+                log.warn("Unable to print contents of key store: {}", ksLocation, e);
             }
         }
     }
@@ -449,18 +506,8 @@
 
         @Override
         protected void initChannel(SocketChannel channel) throws Exception {
-            TrustManagerFactory tmFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-            KeyStore ts = KeyStore.getInstance("JKS");
-            ts.load(new FileInputStream(tsLocation), tsPwd);
-            tmFactory.init(ts);
-
-            KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-            KeyStore ks = KeyStore.getInstance("JKS");
-            ks.load(new FileInputStream(ksLocation), ksPwd);
-            kmf.init(ks, ksPwd);
-
             SSLContext serverContext = SSLContext.getInstance("TLS");
-            serverContext.init(kmf.getKeyManagers(), tmFactory.getTrustManagers(), null);
+            serverContext.init(keyManager.getKeyManagers(), trustManager.getTrustManagers(), null);
 
             SSLEngine serverSslEngine = serverContext.createSSLEngine();
 
@@ -486,18 +533,8 @@
 
         @Override
         protected void initChannel(SocketChannel channel) throws Exception {
-            TrustManagerFactory tmFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-            KeyStore ts = KeyStore.getInstance("JKS");
-            ts.load(new FileInputStream(tsLocation), tsPwd);
-            tmFactory.init(ts);
-
-            KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-            KeyStore ks = KeyStore.getInstance("JKS");
-            ks.load(new FileInputStream(ksLocation), ksPwd);
-            kmf.init(ks, ksPwd);
-
             SSLContext clientContext = SSLContext.getInstance("TLS");
-            clientContext.init(kmf.getKeyManagers(), tmFactory.getTrustManagers(), null);
+            clientContext.init(keyManager.getKeyManagers(), trustManager.getTrustManagers(), null);
 
             SSLEngine clientSslEngine = clientContext.createSSLEngine();
 
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/CopycatTransportClient.java b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/CopycatTransportClient.java
index c88242c..afa98cc 100644
--- a/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/CopycatTransportClient.java
+++ b/core/store/primitives/src/main/java/org/onosproject/store/primitives/impl/CopycatTransportClient.java
@@ -75,6 +75,8 @@
                         if (MessagingException.class.isAssignableFrom(rootCause.getClass())) {
                             wrappedError = new TransportException(error);
                         }
+                        // TODO ONOS-6788 we might consider demoting this warning during startup when there is
+                        //      a race between the server registering handlers and the client sending messages
                         log.warn("Connection to {} failed! Reason: {}", address, wrappedError);
                         future.completeExceptionally(wrappedError);
                     } else {
diff --git a/tools/build/envDefaults b/tools/build/envDefaults
index 41a9503..fd282c9 100644
--- a/tools/build/envDefaults
+++ b/tools/build/envDefaults
@@ -51,3 +51,6 @@
 export ONOS_GROUP="${ONOS_GROUP:-sdn}"  # ONOS group on remote system
 export ONOS_PWD="rocks"                 # ONOS user password on remote system
 export ONOS_SCENARIOS=$ONOS_ROOT/tools/test/scenarios
+
+export ONOS_CLUSTER_KEY_FILE="/tmp/onos.jks"
+export ONOS_CLUSTER_KEY_PASSWORD="changeit"
\ No newline at end of file
diff --git a/tools/test/bin/onos-gen-cluster-key b/tools/test/bin/onos-gen-cluster-key
new file mode 100755
index 0000000..168a174
--- /dev/null
+++ b/tools/test/bin/onos-gen-cluster-key
@@ -0,0 +1,23 @@
+#!/bin/bash
+# ------------------------------------------------------------------------
+# This script generates a self-signed certificate and private key pair
+# and stores them in a Java keystore. This keystore can be used as the
+# keystore and trust store for client and server ends of TLS connections
+# for all nodes in the cluster.
+# ------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+[ "$1" = "-f" ] && shift && generate_new_key=true
+
+[ "$generate_new_key" = true ] && rm -f $ONOS_CLUSTER_KEY_FILE
+
+keytool -genkey -keystore $ONOS_CLUSTER_KEY_FILE \
+        -storepass $ONOS_CLUSTER_KEY_PASSWORD \
+        -keyalg RSA \
+        -alias onos \
+        -validity 3600 \
+        -keysize 2048 \
+        -dname CN=onos \
+        -keypass $ONOS_CLUSTER_KEY_PASSWORD
\ No newline at end of file
diff --git a/tools/test/bin/onos-install b/tools/test/bin/onos-install
index 2231667..cc00da0 100755
--- a/tools/test/bin/onos-install
+++ b/tools/test/bin/onos-install
@@ -102,5 +102,8 @@
 # Configure the ONOS installation
 onos-config $node
 
+# Upload the shared cluster key if present
+[ -f "$ONOS_CLUSTER_KEY_FILE" ] && onos-push-cluster-key $1
+
 # Unless -n option was given, attempt to ignite the ONOS service.
 [ -z "$nostart" ] && onos-service $node start || true
\ No newline at end of file
diff --git a/tools/test/bin/onos-push-cluster-key b/tools/test/bin/onos-push-cluster-key
new file mode 100755
index 0000000..c2a77f6
--- /dev/null
+++ b/tools/test/bin/onos-push-cluster-key
@@ -0,0 +1,11 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Pushes the cluster key to the ONOS config directory on a remote ONOS node.
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+remote=$ONOS_USER@${1:-$OCI}
+
+scp -q $ONOS_CLUSTER_KEY_FILE $remote:$ONOS_INSTALL_DIR/config/onos.jks
\ No newline at end of file
diff --git a/tools/test/scenarios/setup.xml b/tools/test/scenarios/setup.xml
index 4f45deb..1d1c36a 100644
--- a/tools/test/scenarios/setup.xml
+++ b/tools/test/scenarios/setup.xml
@@ -30,19 +30,21 @@
         </group>
 
         <group name="Install">
+            <step name="Generate-Cluster-Key" exec="onos-gen-cluster-key -f" />
+
             <group name="Sequential-Install" if="${ONOS_STC_SEQ_START}">
                 <sequential var="${OC#}"
                             starts="Sequential-Install-${#}"
                             ends="Sequential-Install-${#-1}">
                     <step name="Sequential-Install-${#}" exec="onos-install ${OC#}"
-                          requires="Push-Bits-${#},Push-Bits,Cleanup"/>
+                          requires="Generate-Cluster-Key,Push-Bits-${#},Push-Bits,Cleanup"/>
                 </sequential>
             </group>
 
             <group name="Parallel-Install" unless="${ONOS_STC_SEQ_START}">
                 <parallel var="${OC#}">
                     <step name="Parallel-Install-${#}" exec="onos-install ${OC#}"
-                          requires="Push-Bits-${#},Push-Bits,Cleanup"/>
+                          requires="Generate-Cluster-Key,Push-Bits-${#},Push-Bits,Cleanup"/>
                 </parallel>
             </group>
         </group>