Incorrect maximum encryption size for RSA OAEP (#423)

Incorrect maximum encryption size for RSA OAEP

### Checklist
- [X] I've run tests to see all new and existing tests pass
- [X] I've followed the code style of the rest of the project
- [X] I've read the [Contribution Guidelines](CONTRIBUTING.md)
- [X] I've updated the documentation if necessary

#### If you've made changes to `gyb` files
- [n/a] I've run `./scripts/generate_boilerplate_files_with_gyb.sh` and
included updated generated files in a commit of this pull request

### Motivation:
The `maximumEncryptSize` function is hardcoded to use 42 as the hash
offset, but the RFC actually says it's "2*hLen-2" so 42 is only valid
for SHA1. SHA256 should be 62 (2*32-2). This adds a hash length onto the
Digest enum, which can then be used in the length calculation.

In writing the tests for this, I also stumbled on the unsafe PEM
representation for RSA Public Keys don't allow 1024-bit keys as
documented, so this also fixes that.

### Modifications:
* Added a `hashBitLength` to the RSA Digest which is then used in the
`maximumEncryptSize` to properly compute the maximum length
* Corrected the minimum key size for RSA unsafe PEM public keys to 1024
from 2048

### Result:
* The `maximumEncryptSize` function will return the expected value for
RSA OAEP SHA256 keys. I don't believe anything calls this internally, so
this would be for external consumers.
* Unsafe construction of RSA public keys now allow 1024 bit keys. It was
documented as supporting them, but the check was still 2048 (probably
copy-paste error from the safe variant). This only expands the potential
uses, so it shouldn't introduce any new failures.

Co-authored-by: Mike Schrag <[email protected]>

Author: mschrag

Sources/CryptoExtras/RSA/RSA.swift

@@ -508,7 +508,7 @@ extension _RSA.Encryption {
         /// - Warning: Key sizes less than 2048 are not recommended and should only be used for compatibility reasons.
         public init(unsafePEMRepresentation pemRepresentation: String) throws {
             self.backing = try BackingPublicKey(pemRepresentation: pemRepresentation)
-            guard self.keySizeInBits >= 2048, self.keySizeInBits % 8 == 0 else { throw CryptoKitError.incorrectParameterSize }
+            guard self.keySizeInBits >= 1024, self.keySizeInBits % 8 == 0 else { throw CryptoKitError.incorrectParameterSize }
         }
 
         /// Construct an RSA public key from a DER representation.
@@ -675,6 +675,14 @@ extension _RSA.Encryption {
     internal enum Digest {
         case sha1
         case sha256
+
+        /// Returns the number of bits in the resulting hash
+        var hashBitLength: Int {
+            switch self {
+            case .sha1: return 160
+            case .sha256: return 256
+            }
+        }
     }
 }
 
@@ -694,7 +702,7 @@ extension _RSA.Encryption.PublicKey {
     /// Return the maximum amount of data in bytes this key can encrypt in a single operation when using
     /// the specified padding mode.
     ///
-    /// ## Common values:
+    /// ## Common values (for PKCS1 OAEP SHA1):
     ///
     /// Key size|Padding|Max length
     /// -|-|-
@@ -703,8 +711,9 @@ extension _RSA.Encryption.PublicKey {
     /// 4096|PKCS-OAEP|470 bytes
     public func maximumEncryptSize(with padding: _RSA.Encryption.Padding) -> Int {
         switch padding.backing {
-        case .pkcs1_oaep:
-            return (self.keySizeInBits / 8) - 42
+        case let .pkcs1_oaep(Digest):
+            // https://datatracker.ietf.org/doc/html/rfc8017#section-7.1.1
+            return (self.keySizeInBits / 8) - (2 * Digest.hashBitLength / 8) - 2
         }
     }
 

Tests/CryptoExtrasTests/TestRSAEncryption.swift

@@ -140,6 +140,35 @@ final class TestRSAEncryption: XCTestCase {
             XCTAssertEqual(primitives.publicExponent, e)
         }
     }
+
+    func testMaximumEncryptSize() throws {
+        let pemRepresentation1024 = """
+            -----BEGIN PUBLIC KEY-----
+            MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCTLbu1QZhtXWKCHOjavP5NUCwJ
+            5DwjoMKGlEM/PQOMiY+wup8R1kCOHV6g+FvJ86laHJc0gqwFf1U51YxtQFy7cGV4
+            W2zJeTkqadO2fvTCjbZU+Oa78iVtTynq5h4yRWrTmveyzInhdVpi075Ql2hpGuET
+            H1qYVxqaDIJEHyETDQIDAQAB
+            -----END PUBLIC KEY-----
+            """
+        let pubKey1024 = try _RSA.Encryption.PublicKey(unsafePEMRepresentation: pemRepresentation1024)
+        XCTAssertEqual(86, pubKey1024.maximumEncryptSize(with: .PKCS1_OAEP))
+        XCTAssertEqual(62, pubKey1024.maximumEncryptSize(with: .PKCS1_OAEP_SHA256))
+
+        let pemRepresentation2048 = """
+            -----BEGIN PUBLIC KEY-----
+            MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx8IRKcs5FrHlWye2lfwc
+            Hr0Pi8g5iZhGMOOwmyIVsNULAvUIGZlIw38NNqebH3eF6ZxPiSRpwPwIs6QRcH5/
+            IwbHkUc0KdBbUwXDrLs0w00I7Flu5RP7IEfkOZdDGEWFY1pA3H1HaogxKFc5k3mM
+            s7pW6oty1eP4O7aVa/Pp363Vba7EZ2nru9lz4Ta+JU8UIHbpoddMGikGEKHrQ/Ge
+            n9RMNzSIy/e7TgTwC39GKn8fwN6VfcdNjvIhJrFNha/ORNArpzup7FUUauGLKt3a
+            jgsIjrAPBp63+Sy7+aFVoGTvI7DCkZ/Wv3JCFRuTAdYOa0A1xiqhTb1pcypvrd2T
+            ZQIDAQAB
+            -----END PUBLIC KEY-----
+            """
+        let pubKey2048 = try _RSA.Encryption.PublicKey(pemRepresentation: pemRepresentation2048)
+        XCTAssertEqual(214, pubKey2048.maximumEncryptSize(with: .PKCS1_OAEP))
+        XCTAssertEqual(190, pubKey2048.maximumEncryptSize(with: .PKCS1_OAEP_SHA256))
+    }
 }
 
 struct RSAEncryptionOAEPTestGroup: Codable {