Introduce SslMasterKeyHandler (netty#8653)

Motivation

Debugging SSL/TLS connections through wireshark is a pain -- if the cipher used involves Diffie-Hellman then it is essentially impossible unless you can have the client dump out the master key [1]

This is a work-in-progress change (tests & comments to come!) that introduces a new handler you can set on the SslContext to receive the master key & session id. I'm hoping to get feedback if a change in this vein would be welcomed.

An implementation that conforms to Wireshark's NSS key log[2] file is also included.

Depending on feedback on the PR going forward I am planning to "clean it up" by adding documentation, example server & tests. Implementation will need to be finished as well for retrieving the master key from the OpenSSL context.

[1] https://jimshaver.net/2015/02/11/decrypting-tls-browser-traffic-with-wireshark-the-easy-way/
[2] https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/Key_Log_Format

Modification

- Added SslMasterKeyHandler
- An implementation of the handler that conforms to Wireshark's key log format is included.

Result:

Be able to debug SSL / TLS connections more easily.

Signed-off-by: Farid Zakaria <[email protected]>

Author: fzakaria

handler/src/main/java/io/netty/handler/ssl/SslMasterKeyHandler.java

@@ -0,0 +1,190 @@
+/*
+ * Copyright 2019 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import io.netty.buffer.ByteBufUtil;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.internal.tcnative.SSL;
+import io.netty.util.internal.ReflectionUtil;
+import io.netty.util.internal.SystemPropertyUtil;
+import io.netty.util.internal.logging.InternalLogger;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLSession;
+import java.lang.reflect.Field;
+
+/**
+ * The {@link SslMasterKeyHandler} is a channel-handler you can include in your pipeline to consume the master key
+ * & session identifier for a TLS session.
+ * This can be very useful, for instance the {@link WiresharkSslMasterKeyHandler} implementation will
+ * log the secret & identifier in a format that is consumable by Wireshark -- allowing easy decryption of pcap/tcpdumps.
+ */
+public abstract class SslMasterKeyHandler extends ChannelInboundHandlerAdapter {
+
+    private static final InternalLogger logger = InternalLoggerFactory.getInstance(SslMasterKeyHandler.class);
+
+    /**
+     * The JRE SSLSessionImpl cannot be imported
+     */
+    private static final Class<?> SSL_SESSIONIMPL_CLASS;
+
+    /**
+     * The master key field in the SSLSessionImpl
+     */
+    private static final Field SSL_SESSIONIMPL_MASTER_SECRET_FIELD;
+
+    /**
+     * A system property that can be used to turn on/off the {@link SslMasterKeyHandler} dynamically without having
+     * to edit your pipeline.
+     * <code>-Dio.netty.ssl.masterKeyHandler=true</code>
+     */
+    public static final String SYSTEM_PROP_KEY = "io.netty.ssl.masterKeyHandler";
+
+    /**
+     * The unavailability cause of whether the private Sun implementation of SSLSessionImpl is available.
+     */
+    private static final Throwable UNAVAILABILITY_CAUSE;
+
+    static {
+        Throwable cause = null;
+        Class<?> clazz = null;
+        Field field = null;
+        try {
+            clazz = Class.forName("sun.security.ssl.SSLSessionImpl");
+            field = clazz.getDeclaredField("masterSecret");
+            cause = ReflectionUtil.trySetAccessible(field, true);
+        } catch (Throwable e) {
+            cause = e;
+            logger.debug("sun.security.ssl.SSLSessionImpl is unavailable.", e);
+        }
+        UNAVAILABILITY_CAUSE = cause;
+        SSL_SESSIONIMPL_CLASS = clazz;
+        SSL_SESSIONIMPL_MASTER_SECRET_FIELD = field;
+    }
+
+    /**
+     * Constructor.
+    */
+    protected SslMasterKeyHandler() {
+    }
+
+    /**
+     * Ensure that SSLSessionImpl is available.
+     * @throws UnsatisfiedLinkError if unavailable
+     */
+    public static void ensureSunSslEngineAvailability() {
+        if (UNAVAILABILITY_CAUSE != null) {
+            throw new IllegalStateException(
+                    "Failed to find SSLSessionImpl on classpath", UNAVAILABILITY_CAUSE);
+        }
+    }
+
+    /**
+     * Returns the cause of unavailability.
+     *
+     * @return the cause if unavailable. {@code null} if available.
+     */
+    public static Throwable sunSslEngineUnavailabilityCause() {
+        return UNAVAILABILITY_CAUSE;
+    }
+
+    /* Returns {@code true} if and only if sun.security.ssl.SSLSessionImpl exists in the runtime.
+     */
+    public static boolean isSunSslEngineAvailable() {
+        return UNAVAILABILITY_CAUSE == null;
+    }
+
+    /**
+     * Consume the master key for the session and the sessionId
+     * @param masterKey A 48-byte secret shared between the client and server.
+     * @param session The current TLS session
+     */
+    protected abstract void accept(SecretKey masterKey, SSLSession session);
+
+    @Override
+    public final void userEventTriggered(ChannelHandlerContext ctx, Object evt) {
+        //only try to log the session info if the ssl handshake has successfully completed.
+        if (evt == SslHandshakeCompletionEvent.SUCCESS) {
+            boolean shouldHandle = SystemPropertyUtil.getBoolean(SYSTEM_PROP_KEY, false);
+
+            if (shouldHandle) {
+                final SslHandler handler = ctx.pipeline().get(SslHandler.class);
+                final SSLEngine engine = handler.engine();
+                final SSLSession sslSession = engine.getSession();
+
+                //the OpenJDK does not expose a way to get the master secret, so try to use reflection to get it.
+                if (isSunSslEngineAvailable() && sslSession.getClass().equals(SSL_SESSIONIMPL_CLASS)) {
+                    final SecretKey secretKey;
+                    try {
+                        secretKey = (SecretKey) SSL_SESSIONIMPL_MASTER_SECRET_FIELD.get(sslSession);
+                    } catch (IllegalAccessException e) {
+                        throw new IllegalArgumentException("Failed to access the field 'masterSecret' " +
+                                "via reflection.", e);
+                    }
+                    accept(secretKey, sslSession);
+                } else if (OpenSsl.isAvailable() && engine instanceof ReferenceCountedOpenSslEngine) {
+                    SecretKeySpec secretKey = new SecretKeySpec(
+                            SSL.getMasterKey(((ReferenceCountedOpenSslEngine) engine).sslPointer()), "AES");
+                    accept(secretKey, sslSession);
+                }
+            }
+        }
+
+        ctx.fireUserEventTriggered(evt);
+    }
+
+    /**
+     * Create a {@link WiresharkSslMasterKeyHandler} instance.
+     * This TLS master key handler logs the master key and session-id in a format
+     * understood by Wireshark -- this can be especially useful if you need to ever
+     * decrypt a TLS session and are using perfect forward secrecy (i.e. Diffie-Hellman)
+     * The key and session identifier are forwarded to the log named 'io.netty.wireshark'.
+     */
+    public static SslMasterKeyHandler newWireSharkSslMasterKeyHandler() {
+        return new WiresharkSslMasterKeyHandler();
+    }
+
+    /**
+     * Record the session identifier and master key to the {@link InternalLogger} named <code>io.netty.wireshark</code>.
+     * ex. <code>RSA Session-ID:XXX Master-Key:YYY</code>
+     * This format is understood by Wireshark 1.6.0.
+     * https://code.wireshark.org/review/gitweb?p=wireshark.git;a=commit;h=686d4cabb41185591c361f9ec6b709034317144b
+     * The key and session identifier are forwarded to the log named 'io.netty.wireshark'.
+     */
+    private static final class WiresharkSslMasterKeyHandler extends SslMasterKeyHandler {
+
+        private static final InternalLogger wireshark_logger =
+                InternalLoggerFactory.getInstance("io.netty.wireshark");
+
+        private static final char[] hexCode = "0123456789ABCDEF".toCharArray();
+
+        @Override
+        protected void accept(SecretKey masterKey, SSLSession session) {
+            if (masterKey.getEncoded().length != 48) {
+                throw new IllegalArgumentException("An invalid length master key was provided.");
+            }
+            final byte[] sessionId = session.getId();
+            wireshark_logger.warn("RSA Session-ID:{} Master-Key:{}",
+                    ByteBufUtil.hexDump(sessionId).toLowerCase(),
+                    ByteBufUtil.hexDump(masterKey.getEncoded()).toLowerCase());
+        }
+    }
+
+}

handler/src/test/java/io/netty/handler/ssl/SSLEngineTest.java

@@ -46,6 +46,8 @@
 import io.netty.util.internal.EmptyArrays;
 import io.netty.util.internal.PlatformDependent;
 import io.netty.util.internal.StringUtil;
+import io.netty.util.internal.SystemPropertyUtil;
+import org.conscrypt.OpenSSLProvider;
 import org.junit.After;
 import org.junit.Assume;
 import org.junit.Before;
@@ -55,10 +57,12 @@
 import org.mockito.MockitoAnnotations;
 
 import java.io.ByteArrayInputStream;
+import java.io.Closeable;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.nio.ByteBuffer;
@@ -84,12 +88,14 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+import javax.crypto.SecretKey;
 import javax.net.ssl.ExtendedSSLSession;
 import javax.net.ssl.KeyManager;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.KeyManagerFactorySpi;
 import javax.net.ssl.ManagerFactoryParameters;
 import javax.net.ssl.SNIHostName;
+import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLEngineResult.Status;
@@ -3245,6 +3251,89 @@ protected TrustManager[] engineGetTrustManagers() {
         }
     }
 
+    @Test
+    public void testMasterKeyLogging() throws Exception {
+
+        /*
+         * At the moment master key logging is not supported for conscrypt
+         */
+        Assume.assumeFalse(serverSslContextProvider() instanceof OpenSSLProvider);
+
+        /*
+         * The JDK SSL engine master key retrieval relies on being able to set field access to true.
+         * That is not available in JDK9+
+         */
+        Assume.assumeFalse(sslServerProvider() == SslProvider.JDK && PlatformDependent.javaVersion() > 8);
+
+        String originalSystemPropertyValue = SystemPropertyUtil.get(SslMasterKeyHandler.SYSTEM_PROP_KEY);
+        System.setProperty(SslMasterKeyHandler.SYSTEM_PROP_KEY, Boolean.TRUE.toString());
+
+        SelfSignedCertificate ssc = new SelfSignedCertificate();
+        serverSslCtx = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey())
+                .sslProvider(sslServerProvider())
+                .sslContextProvider(serverSslContextProvider())
+                .build();
+        Socket socket = null;
+
+        try {
+            sb = new ServerBootstrap();
+            sb.group(new NioEventLoopGroup(), new NioEventLoopGroup());
+            sb.channel(NioServerSocketChannel.class);
+
+            final Promise<SecretKey> promise = sb.config().group().next().newPromise();
+            serverChannel = sb.childHandler(new ChannelInitializer<Channel>() {
+                @Override
+                protected void initChannel(Channel ch) throws Exception {
+                    ch.config().setAllocator(new TestByteBufAllocator(ch.config().getAllocator(), type));
+
+                    SslHandler sslHandler = delegatingExecutor == null ?
+                            serverSslCtx.newHandler(ch.alloc()) :
+                            serverSslCtx.newHandler(ch.alloc(), delegatingExecutor);
+
+                    ch.pipeline().addLast(sslHandler);
+                    ch.pipeline().addLast(new SslMasterKeyHandler() {
+                        @Override
+                        protected void accept(SecretKey masterKey, SSLSession session) {
+                            promise.setSuccess(masterKey);
+                        }
+                    });
+                    serverConnectedChannel = ch;
+                }
+            }).bind(new InetSocketAddress(0)).sync().channel();
+
+            int port = ((InetSocketAddress) serverChannel.localAddress()).getPort();
+
+            SSLContext sslContext = SSLContext.getInstance("TLS");
+            sslContext.init(null, InsecureTrustManagerFactory.INSTANCE.getTrustManagers(), null);
+            socket = sslContext.getSocketFactory().createSocket(NetUtil.LOCALHOST, port);
+            OutputStream out = socket.getOutputStream();
+            out.write(1);
+            out.flush();
+
+            assertTrue(promise.await(10, TimeUnit.SECONDS));
+            SecretKey key = promise.get();
+            assertEquals("AES secret key must be 48 bytes", 48, key.getEncoded().length);
+        } finally {
+            closeQuietly(socket);
+            if (originalSystemPropertyValue != null) {
+                System.setProperty(SslMasterKeyHandler.SYSTEM_PROP_KEY, originalSystemPropertyValue);
+            } else {
+                System.clearProperty(SslMasterKeyHandler.SYSTEM_PROP_KEY);
+            }
+            ssc.delete();
+        }
+    }
+
+    private static void closeQuietly(Closeable c) {
+        if (c != null) {
+            try {
+                c.close();
+            } catch (IOException ignore) {
+                // ignore
+            }
+        }
+    }
+
     private KeyManagerFactory newKeyManagerFactory(SelfSignedCertificate ssc)
             throws UnrecoverableKeyException, KeyStoreException, NoSuchAlgorithmException,
             CertificateException, IOException {