Cherrypick hardening DSA param checks from BoringSSL (#2293)

### Issues:
`P211191334`

### Description of changes: 
This PR cherrypicks 68c29a24ee6c9c70ecce56766ca70b115aad768f from
BoringSSL.
This change adds size checks to DSA_generate_parameters_ex and
DSA_generate_key operations, completing our bounds checks for DSA
operations. While this was prompted by CVE-2024-4603, BoringSSL and
AWS-LC are not affected by that vulnerability. These additional checks
likely have no security impact since attackers typically can't control
inputs to these functions, but we're adding them for consistency with
our other DSA operations.

### Call-outs:
Point out areas that need special attention or support during the review
process. Discuss architecture or design changes.

### Testing:
How is this change tested (unit tests, fuzz tests, etc.)? Are there any
testing steps to be verified by the reviewer?

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.

Co-authored-by: David Benjamin <[email protected]>

Author: smittals2

crypto/dsa/dsa.c

@@ -241,6 +241,11 @@ int dsa_internal_paramgen(DSA *dsa, size_t bits, const EVP_MD *evpmd,
                           int *out_counter, unsigned long *out_h,
                           BN_GENCB *cb)
 {
+  if (bits > OPENSSL_DSA_MAX_MODULUS_BITS) {
+    OPENSSL_PUT_ERROR(DSA, DSA_R_INVALID_PARAMETERS);
+    return 0;
+  }
+
   int ok = 0;
   unsigned char seed[SHA256_DIGEST_LENGTH];
   unsigned char md[SHA256_DIGEST_LENGTH];
@@ -510,11 +515,13 @@ DSA *DSAparams_dup(const DSA *dsa) {
 }
 
 int DSA_generate_key(DSA *dsa) {
+  if (!dsa_check_key(dsa)) {
+    return 0;
+  }
+
   int ok = 0;
-  BN_CTX *ctx = NULL;
   BIGNUM *pub_key = NULL, *priv_key = NULL;
-
-  ctx = BN_CTX_new();
+  BN_CTX *ctx = BN_CTX_new();
   if (ctx == NULL) {
     goto err;
   }

crypto/dsa/dsa_asn1.c

@@ -66,8 +66,6 @@
 #include "../crypto/internal.h"
 
 
-#define OPENSSL_DSA_MAX_MODULUS_BITS 10000
-
 // This function is in dsa_asn1.c rather than dsa.c because it is reachable from
 // |EVP_PKEY| parsers. This makes it easier for the static linker to drop most
 // of the DSA implementation.

crypto/dsa/dsa_test.cc

@@ -236,6 +236,30 @@ TEST(DSATest, Generate) {
                           sig_len, dsa.get()));
 }
 
+TEST(DSATest, GenerateParamsTooLarge) {
+  bssl::UniquePtr<DSA> dsa(DSA_new());
+  ASSERT_TRUE(dsa);
+  EXPECT_FALSE(DSA_generate_parameters_ex(
+      dsa.get(), 10001, /*seed=*/nullptr, /*seed_len=*/0,
+      /*out_counter=*/nullptr, /*out_h=*/nullptr,
+      /*cb=*/nullptr));
+}
+
+TEST(DSATest, GenerateKeyTooLarge) {
+  bssl::UniquePtr<DSA> dsa = GetFIPSDSA();
+  ASSERT_TRUE(dsa);
+  bssl::UniquePtr<BIGNUM> large_p(BN_new());
+  ASSERT_TRUE(large_p);
+  ASSERT_TRUE(BN_set_bit(large_p.get(), 10001));
+  ASSERT_TRUE(BN_set_bit(large_p.get(), 0));
+  ASSERT_TRUE(DSA_set0_pqg(dsa.get(), /*p=*/large_p.get(), /*q=*/nullptr,
+                           /*g=*/nullptr));
+  large_p.release();  // |DSA_set0_pqg| takes ownership on success.
+
+  // Don't generate DSA keys if the group is too large.
+  EXPECT_FALSE(DSA_generate_key(dsa.get()));
+}
+
 TEST(DSATest, Verify) {
   bssl::UniquePtr<DSA> dsa = GetFIPSDSA();
   ASSERT_TRUE(dsa);

crypto/dsa/internal.h

@@ -40,6 +40,8 @@ struct dsa_st {
   CRYPTO_EX_DATA ex_data;
 };
 
+#define OPENSSL_DSA_MAX_MODULUS_BITS 10000
+
 // dsa_check_key performs cheap self-checks on |dsa|, and ensures it is within
 // DoS bounds. It returns one on success and zero on error.
 int dsa_check_key(const DSA *dsa);