[release/6.0] Fix OpenSSL 3 reporting an OutOfMemoryException for missing private key

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.cs

@@ -105,18 +105,28 @@ internal static unsafe SafeEvpPKeyHandle DecodePkcs8PrivateKey(
         }
 
         [DllImport(Libraries.CryptoNative)]
-        private static extern int CryptoNative_GetPkcs8PrivateKeySize(IntPtr pkey);
+        private static extern int CryptoNative_GetPkcs8PrivateKeySize(IntPtr pkey, out int p8size);
 
         private static int GetPkcs8PrivateKeySize(IntPtr pkey)
         {
-            int ret = CryptoNative_GetPkcs8PrivateKeySize(pkey);
+            const int Success = 1;
+            const int Error = -1;
+            const int MissingPrivateKey = -2;
 
-            if (ret < 0)
+            int ret = CryptoNative_GetPkcs8PrivateKeySize(pkey, out int p8size);
+
+            switch (ret)
             {
-                throw CreateOpenSslCryptographicException();
+                case Success:
+                    return p8size;
+                case Error:
+                    throw CreateOpenSslCryptographicException();
+                case MissingPrivateKey:
+                    throw new CryptographicException(SR.Cryptography_CSP_NoPrivateKey);
+                default:
+                    Debug.Fail($"Unexpected return '{ret}' value from {nameof(CryptoNative_GetPkcs8PrivateKeySize)}.");
+                    throw new CryptographicException();
             }
-
-            return ret;
         }
 
         [DllImport(Libraries.CryptoNative)]

src/libraries/Native/Unix/System.Security.Cryptography.Native/opensslshim.h

@@ -270,6 +270,7 @@ const EVP_CIPHER* EVP_chacha20_poly1305(void);
     LEGACY_FUNCTION(ERR_load_crypto_strings) \
     LIGHTUP_FUNCTION(ERR_new) \
     REQUIRED_FUNCTION(ERR_peek_error) \
+    REQUIRED_FUNCTION(ERR_peek_error_line) \
     REQUIRED_FUNCTION(ERR_peek_last_error) \
     FALLBACK_FUNCTION(ERR_put_error) \
     REQUIRED_FUNCTION(ERR_reason_error_string) \
@@ -719,6 +720,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define ERR_load_crypto_strings ERR_load_crypto_strings_ptr
 #define ERR_new ERR_new_ptr
 #define ERR_peek_error ERR_peek_error_ptr
+#define ERR_peek_error_line ERR_peek_error_line_ptr
 #define ERR_peek_last_error ERR_peek_last_error_ptr
 #define ERR_put_error ERR_put_error_ptr
 #define ERR_reason_error_string ERR_reason_error_string_ptr

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey.c

@@ -152,20 +152,59 @@ EVP_PKEY* CryptoNative_DecodePkcs8PrivateKey(const uint8_t* buf, int32_t len, in
     return key;
 }
 
-int32_t CryptoNative_GetPkcs8PrivateKeySize(EVP_PKEY* pkey)
+int32_t CryptoNative_GetPkcs8PrivateKeySize(EVP_PKEY* pkey, int32_t* p8size)
 {
     assert(pkey != NULL);
+    assert(p8size != NULL);
+
+    *p8size = 0;
+    ERR_clear_error();
 
     PKCS8_PRIV_KEY_INFO* p8 = EVP_PKEY2PKCS8(pkey);
 
     if (p8 == NULL)
     {
+        // OpenSSL 1.1 and 3 have a behavioral change with EVP_PKEY2PKCS8
+        // with regard to handling EVP_PKEYs that do not contain a private key.
+        //
+        // In OpenSSL 1.1, it would always succeed, but the private parameters
+        // would be missing (thus making an invalid PKCS8 structure).
+        // Over in the managed side, we detect these invalid PKCS8 blobs and
+        // convert that to a "no private key" error.
+        //
+        // In OpenSSL 3, this now correctly errors, with the error
+        // ASN1_R_ILLEGAL_ZERO_CONTENT. We want to preserve allocation failures
+        // as OutOfMemoryException. So we peek at the error. If it's a malloc
+        // failure, -1 is returned to indcate "throw what is on the error queue".
+        // If the error is not a malloc failure, return -2 to mean "no private key".
+        // If OpenSSL ever changes the error to something more to explicitly mean
+        // "no private key" then we should test for that explicitly. Until then,
+        // we treat all errors, except a malloc error, to mean "no private key".
+
+        const char* file = NULL;
+        int line = 0;
+        unsigned long error = ERR_peek_error_line(&file, &line);
+
+        // If it's not a malloc failure, assume it's because the private key is
+        // missing.
+        if (ERR_GET_REASON(error) != ERR_R_MALLOC_FAILURE)
+        {
+            ERR_clear_error();
+            return -2;
+        }
+
+        // It is a malloc failure. Clear the error queue and set the error
+        // as a malloc error so it's the only error in the queue.
+        ERR_clear_error();
+        ERR_put_error(ERR_GET_LIB(error), 0, ERR_R_MALLOC_FAILURE, file, line);
+
         return -1;
     }
 
-    int ret = i2d_PKCS8_PRIV_KEY_INFO(p8, NULL);
+    *p8size = i2d_PKCS8_PRIV_KEY_INFO(p8, NULL);
     PKCS8_PRIV_KEY_INFO_free(p8);
-    return ret;
+
+    return *p8size < 0 ? -1 : 1;
 }
 
 int32_t CryptoNative_EncodePkcs8PrivateKey(EVP_PKEY* pkey, uint8_t* buf)

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey.h

@@ -58,9 +58,13 @@ Requres a non-null buf, and len > 0.
 PALEXPORT EVP_PKEY* CryptoNative_DecodePkcs8PrivateKey(const uint8_t* buf, int32_t len, int32_t algId);
 
 /*
-Reports the number of bytes rqeuired to encode an EVP_PKEY* as a Pkcs8PrivateKeyInfo, or a negative value on error.
+Gets the number of bytes rqeuired to encode an EVP_PKEY* as a Pkcs8PrivateKeyInfo.
+
+On success, 1 is returned and p8size contains the size of the Pkcs8PrivateKeyInfo.
+On failure, -1 is used to indicate the openssl error queue contains the error.
+On failure, -2 is used to indcate that the supplied EVP_PKEY* is possibly missing a private key.
 */
-PALEXPORT int32_t CryptoNative_GetPkcs8PrivateKeySize(EVP_PKEY* pkey);
+PALEXPORT int32_t CryptoNative_GetPkcs8PrivateKeySize(EVP_PKEY* pkey, int32_t* p8size);
 
 /*
 Encodes the EVP_PKEY* as a Pkcs8PrivateKeyInfo, writing the encoded value to buf.