Handle zero value modulus for OpenSSL 1.1

src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/ImportExport.cs

@@ -311,6 +311,22 @@ public static void ExportAfterDispose(bool importKey)
             }
         }
 
+        [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindows7))]
+        [InlineData(true)]
+        [InlineData(false)]
+        public static void ImportZeroModulus(bool includePrivateParameters)
+        {
+            RSAParameters zeroModulus = CopyRSAParameters(TestData.RSA2048Params);
+            zeroModulus.Modulus.AsSpan().Clear();
+
+            if (!includePrivateParameters)
+            {
+                zeroModulus = MakePublic(zeroModulus);
+            }
+
+            Assert.ThrowsAny<CryptographicException>(() => RSAFactory.Create(zeroModulus));
+        }
+
         internal static void AssertKeyEquals(in RSAParameters expected, in RSAParameters actual)
         {
             Assert.Equal(expected.Modulus, actual.Modulus);
@@ -444,5 +460,22 @@ private static bool TestRsa16384()
                 return false;
             }
         }
+
+        private static RSAParameters CopyRSAParameters(in RSAParameters rsaParams)
+        {
+            static byte[] CopyBytes(byte[] data) => data is null ? null : data.AsSpan().ToArray();
+
+            return new RSAParameters
+            {
+                Modulus = CopyBytes(rsaParams.Modulus),
+                Exponent = CopyBytes(rsaParams.Exponent),
+                D = CopyBytes(rsaParams.D),
+                P = CopyBytes(rsaParams.P),
+                Q = CopyBytes(rsaParams.Q),
+                DP = CopyBytes(rsaParams.DP),
+                DQ = CopyBytes(rsaParams.DQ),
+                InverseQ = CopyBytes(rsaParams.InverseQ),
+            };
+        }
     }
 }

src/native/libs/System.Security.Cryptography.Native/apibridge.c

@@ -890,12 +890,17 @@ int local_EVP_PKEY_public_check(EVP_PKEY_CTX* ctx)
     }
 }
 
-
 int local_ASN1_TIME_to_tm(const ASN1_TIME* s, struct tm* tm)
 {
     (void)s;
     (void)tm;
 
     return 0;
 }
+
+int local_BN_is_zero(const BIGNUM* a)
+{
+    return a->top == 0;
+}
+
 #endif

src/native/libs/System.Security.Cryptography.Native/apibridge.h

@@ -7,6 +7,7 @@
 #include "pal_types.h"
 
 int local_ASN1_TIME_to_tm(const ASN1_TIME* s, struct tm* tm);
+int local_BN_is_zero(const BIGNUM* a);
 int local_BIO_up_ref(BIO *a);
 const BIGNUM* local_DSA_get0_key(const DSA* dsa, const BIGNUM** pubKey, const BIGNUM** privKey);
 void local_DSA_get0_pqg(const DSA* dsa, const BIGNUM** p, const BIGNUM** q, const BIGNUM** g);

src/native/libs/System.Security.Cryptography.Native/openssl_1_0_structs.h

@@ -184,3 +184,11 @@ struct bio_st
     const void*_ignored11;
     int references;
 };
+
+struct bignum_st {
+    const void* _ignored1;
+    int top;
+    int _ignored2;
+    int _ignored3;
+    int _ignored4;
+};

src/native/libs/System.Security.Cryptography.Native/opensslshim.h

@@ -192,6 +192,7 @@ const EVP_CIPHER* EVP_chacha20_poly1305(void);
     REQUIRED_FUNCTION(BN_clear_free) \
     REQUIRED_FUNCTION(BN_dup) \
     REQUIRED_FUNCTION(BN_free) \
+    FALLBACK_FUNCTION(BN_is_zero) \
     REQUIRED_FUNCTION(BN_new) \
     REQUIRED_FUNCTION(BN_num_bits) \
     REQUIRED_FUNCTION(BN_set_word) \
@@ -673,6 +674,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define BN_clear_free BN_clear_free_ptr
 #define BN_dup BN_dup_ptr
 #define BN_free BN_free_ptr
+#define BN_is_zero BN_is_zero_ptr
 #define BN_new BN_new_ptr
 #define BN_num_bits BN_num_bits_ptr
 #define BN_set_word BN_set_word_ptr
@@ -1185,6 +1187,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 
 // Alias "future" API to the local_ version.
 #define ASN1_TIME_to_tm local_ASN1_TIME_to_tm
+#define BN_is_zero local_BN_is_zero
 #define BIO_up_ref local_BIO_up_ref
 #define DSA_get0_key local_DSA_get0_key
 #define DSA_get0_pqg local_DSA_get0_pqg

src/native/libs/System.Security.Cryptography.Native/osslcompat_111.h

@@ -6,6 +6,7 @@
 #pragma once
 #include "pal_types.h"
 
+#undef BN_is_zero
 #undef SSL_CTX_set_options
 #undef SSL_set_options
 #undef SSL_session_reused
@@ -20,6 +21,7 @@ typedef struct stack_st OPENSSL_STACK;
 #define OPENSSL_INIT_LOAD_SSL_STRINGS 0x00200000L
 
 int ASN1_TIME_to_tm(const ASN1_TIME* s, struct tm* tm);
+int BN_is_zero(const BIGNUM* a);
 int BIO_up_ref(BIO* a);
 const BIGNUM* DSA_get0_key(const DSA* dsa, const BIGNUM** pubKey, const BIGNUM** privKey);
 void DSA_get0_pqg(const DSA* dsa, const BIGNUM** p, const BIGNUM** q, const BIGNUM** g);

src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.c

@@ -97,6 +97,26 @@ static bool CheckKey(EVP_PKEY* key, int32_t algId, int32_t (*check_func)(EVP_PKE
         return false;
     }
 
+    // OpenSSL 1.x does not fail when importing a key with a zero modulus. It fails at key-usage time with an
+    // out-of-memory error. For RSA keys, check the modulus for zero and report an invalid key.
+    // OpenSSL 3 correctly fails with with an invalid modulus error.
+    if (algId == NID_rsaEncryption)
+    {
+        const RSA* rsa = EVP_PKEY_get0_RSA(key);
+
+        if (rsa != NULL)
+        {
+            const BIGNUM* modulus = NULL;
+            RSA_get0_key(rsa, &modulus, NULL, NULL);
+
+            if (modulus != NULL && BN_is_zero(modulus))
+            {
+                ERR_put_error(ERR_LIB_EVP, 0, EVP_R_DECODE_ERROR, __FILE__, __LINE__);
+                return false;
+            }
+        }
+    }
+
     EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new(key, NULL);
 
     if (ctx == NULL)