Fix test_pkey_rsa.rb in FIPS.

* Fix test_new.
* Fix test_s_generate.
* Fix test_new_break.
* Fix test_sign_verify.
  Note that I created the signature text (`signature_encoded.txt`), that is used
  as a text to create the `signature0` in the `test_sign_verify` by the following
  steps with the `openssl` CLI on FIPS module.
  ```
  $ OPENSSL_DIR="${HOME}/.local/openssl-3.4.0-dev-fips-debug-3c6e114959"
  $ export OPENSSL_CONF="${OPENSSL_DIR}/ssl/openssl_fips.cnf"

  $ echo -n "Sign me!" > data.txt
  $ "${OPENSSL_DIR}/bin/openssl" dgst -sha256 -sign test/openssl/fixtures/pkey/rsa2048.pem data.txt > signature.txt
  $ cat signature.txt | base64 > signature_encoded.txt
  ```
* Fix test_sign_verify_options.
* Fix test_sign_verify_pss. (WIP)

Author: junaruga

Rakefile

@@ -32,6 +32,7 @@ Rake::TestTask.new(:test_fips_internal) do |t|
     'test/openssl/test_pkey_dh.rb',
     'test/openssl/test_pkey_dsa.rb',
     'test/openssl/test_pkey_ec.rb',
+    'test/openssl/test_pkey_rsa.rb',
   ]
   t.warning = true
 end

test/openssl/test_pkey_rsa.rb

@@ -15,7 +15,15 @@ def test_no_private_exp
 
   def test_private
     # Generated by key size and public exponent
-    key = OpenSSL::PKey::RSA.new(512, 3)
+    #
+    # SP800 requires ossl_ifc_ffc_compute_security_bits that the return value
+    # (strength in bits) is more than equal RSA_FIPS1864_MIN_KEYGEN_STRENGTH
+    # (112) in FIPS.
+    # https://github.com/openssl/openssl/blob/3c6e11495975a4eda4cc5886080afed6203711ac/crypto/rsa/rsa_sp800_56b_gen.c#L176-L182
+    # The ossl_ifc_ffc_compute_security_bits returns 112 with the argument nbits
+    # 2048.
+    # https://github.com/openssl/openssl/blob/3c6e11495975a4eda4cc5886080afed6203711ac/crypto/rsa/rsa_lib.c#L334-L335
+    key = OpenSSL::PKey::RSA.new(2048, 65537)
     assert(key.private?)
 
     # Generated by DER
@@ -46,63 +54,68 @@ def test_private
   end
 
   def test_new
-    key = OpenSSL::PKey::RSA.new(512)
-    assert_equal 512, key.n.num_bits
+    key = OpenSSL::PKey::RSA.new(2048)
+    assert_equal 2048, key.n.num_bits
     assert_equal 65537, key.e
     assert_not_nil key.d
 
     # Specify public exponent
-    key2 = OpenSSL::PKey::RSA.new(512, 3)
-    assert_equal 512, key2.n.num_bits
-    assert_equal 3, key2.e
+    key2 = OpenSSL::PKey::RSA.new(2048, 65537)
+    assert_equal 2048, key2.n.num_bits
+    assert_equal 65537, key2.e
     assert_not_nil key2.d
   end
 
   def test_s_generate
-    key1 = OpenSSL::PKey::RSA.generate(512)
-    assert_equal 512, key1.n.num_bits
+    key1 = OpenSSL::PKey::RSA.generate(2048)
+    assert_equal 2048, key1.n.num_bits
     assert_equal 65537, key1.e
 
     # Specify public exponent
-    key2 = OpenSSL::PKey::RSA.generate(512, 3)
-    assert_equal 512, key2.n.num_bits
-    assert_equal 3, key2.e
+    key2 = OpenSSL::PKey::RSA.generate(2048, 65537)
+    assert_equal 2048, key2.n.num_bits
+    assert_equal 65537, key2.e
     assert_not_nil key2.d
   end
 
   def test_new_break
-    assert_nil(OpenSSL::PKey::RSA.new(1024) { break })
+    assert_nil(OpenSSL::PKey::RSA.new(2048) { break })
     assert_raise(RuntimeError) do
-      OpenSSL::PKey::RSA.new(1024) { raise }
+      OpenSSL::PKey::RSA.new(2048) { raise }
     end
   end
 
   def test_sign_verify
-    rsa1024 = Fixtures.pkey("rsa1024")
+    # The ossl_rsa_check_key_size called in ossl_pkey_sign requires more than
+    # equal 2048 bits on the argument protect = 1 in FIPS.
+    # https://github.com/openssl/openssl/blob/3c6e11495975a4eda4cc5886080afed6203711ac/providers/common/securitycheck.c#L68-L69
+    rsa = Fixtures.pkey("rsa2048")
     data = "Sign me!"
-    signature = rsa1024.sign("SHA256", data)
-    assert_equal true, rsa1024.verify("SHA256", signature, data)
+    signature = rsa.sign("SHA256", data)
+    assert_equal true, rsa.verify("SHA256", signature, data)
 
     signature0 = (<<~'end;').unpack1("m")
-      oLCgbprPvfhM4pjFQiDTFeWI9Sk+Og7Nh9TmIZ/xSxf2CGXQrptlwo7NQ28+
-      WA6YQo8jPH4hSuyWIM4Gz4qRYiYRkl5TDMUYob94zm8Si1HxEiS9354tzvqS
-      zS8MLW2BtNPuTubMxTItHGTnOzo9sUg0LAHVFt8kHG2NfKAw/gQ=
+      ooy49i8aeFtkDYUU0RPDsEugGiNw4lZxpbQPnIwtdftEkka945IqKZ/MY3YSw7wKsvBZeaTy8GqL
+      lSWLThsRFDV+UUS9zUBbQ9ygNIT8OjdV+tNL63ZpKGprczSnw4F05MQIpajNRud/8jiI9rf+Wysi
+      WwXecjMl2FlXlLJHY4PFQZU5TiametB4VCQRMcjLo1uf26u/yRpiGaYyqn5vxs0SqNtUDM1UL6x4
+      NHCAdqLjuFRQPjYp1vGLD3eSl4061pS8x1NVap3YGbYfGUyzZO4VfwFwf1jPdhp/OX/uZw4dGB2H
+      gSK+q1JiDFwEE6yym5tdKovL1g1NhFYHF6gkZg==
     end;
-    assert_equal true, rsa1024.verify("SHA256", signature0, data)
+    assert_equal true, rsa.verify("SHA256", signature0, data)
     signature1 = signature0.succ
-    assert_equal false, rsa1024.verify("SHA256", signature1, data)
+    assert_equal false, rsa.verify("SHA256", signature1, data)
   end
 
   def test_sign_verify_options
-    key = Fixtures.pkey("rsa1024")
+    key = Fixtures.pkey("rsa2048")
     data = "Sign me!"
     pssopts = {
       "rsa_padding_mode" => "pss",
       "rsa_pss_saltlen" => 20,
       "rsa_mgf1_md" => "SHA1"
     }
     sig_pss = key.sign("SHA256", data, pssopts)
-    assert_equal 128, sig_pss.bytesize
+    assert_equal 256, sig_pss.bytesize
     assert_equal true, key.verify("SHA256", sig_pss, data, pssopts)
     assert_equal true, key.verify_pss("SHA256", sig_pss, data,
                                       salt_length: 20, mgf1_hash: "SHA1")
@@ -175,12 +188,12 @@ def test_verify_empty_rsa
   end
 
   def test_sign_verify_pss
-    key = Fixtures.pkey("rsa1024")
+    key = Fixtures.pkey("rsa2048")
     data = "Sign me!"
     invalid_data = "Sign me?"
 
     signature = key.sign_pss("SHA256", data, salt_length: 20, mgf1_hash: "SHA1")
-    assert_equal 128, signature.bytesize
+    assert_equal 256, signature.bytesize
     assert_equal true,
       key.verify_pss("SHA256", signature, data, salt_length: 20, mgf1_hash: "SHA1")
     assert_equal true,