ossl_pkey.c: Workaround: Decode with non-zero selections.

This is a workaround for the decoding issue in ossl_pkey_read_generic().
The issue happens in the case that a key management provider is different from
a decoding provider.

Try all the non-zero selections EVP_PKEY_KEY_PARAMETERS, EVP_PKEY_PUBLIC_KEY
and EVP_PKEY_PUBLIC_KEY in order, instead of selection 0 for OpenSSL 3 to avoid
the issue.

Author: junaruga

ext/openssl/ossl_pkey.c

@@ -83,23 +83,69 @@ ossl_pkey_new(EVP_PKEY *pkey)
 # include <openssl/decoder.h>
 
 EVP_PKEY *
-ossl_pkey_read_generic(BIO *bio, VALUE pass)
+ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass)
 {
     void *ppass = (void *)pass;
-    OSSL_DECODER_CTX *dctx;
+    OSSL_DECODER_CTX *dctx = NULL;
     EVP_PKEY *pkey = NULL;
-    int pos = 0, pos2;
+    int pos = 0, pos2 = 0;
 
-    dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "DER", NULL, NULL, 0, NULL, NULL);
+    dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, input_type, NULL, NULL,
+                                         selection, NULL, NULL);
     if (!dctx)
         goto out;
-    if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
+    if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb,
+                                             ppass) != 1)
         goto out;
+    while (1) {
+        if (OSSL_DECODER_from_bio(dctx, bio) == 1)
+            goto out;
+        if (BIO_eof(bio))
+            break;
+        pos2 = BIO_tell(bio);
+        if (pos2 < 0 || pos2 <= pos)
+            break;
+        ossl_clear_error();
+        pos = pos2;
+    }
+  out:
+    OSSL_BIO_reset(bio);
+    OSSL_DECODER_CTX_free(dctx);
+    return pkey;
+}
+
+EVP_PKEY *
+ossl_pkey_read_generic(BIO *bio, VALUE pass)
+{
+    EVP_PKEY *pkey = NULL;
+    /* Non-zero selections to try to decode.
+     *
+     * See EVP_PKEY_fromdata(3) - Selections to see all the selections.
+     *
+     * This is a workaround for the decoder failing to decode or returning
+     * bogus keys with selection 0, if a key management provider is different
+     * from a decoder provider. The workaround is to avoid using selection 0.
+     *
+     * Affected OpenSSL versions: >= 3.1.0, <= 3.1.2, or >= 3.0.0, <= 3.0.10
+     * Fixed OpenSSL versions: 3.2, next release of the 3.1.z and 3.0.z
+     *
+     * See <https://github.com/openssl/openssl/pull/21519> for details.
+     */
+    int selections[] = {
+        EVP_PKEY_KEY_PARAMETERS,
+        EVP_PKEY_PUBLIC_KEY,
+        EVP_PKEY_KEYPAIR
+    };
+    int i = 0;
+    int selection_num = sizeof(selections) / sizeof(int);
 
     /* First check DER */
-    if (OSSL_DECODER_from_bio(dctx, bio) == 1)
-        goto out;
-    OSSL_BIO_reset(bio);
+    for (i = 0; i < selection_num; i++) {
+        pkey = ossl_pkey_read(bio, "DER", selections[i], pass);
+        if (pkey) {
+            goto out;
+        }
+    }
 
     /*
      * Then check PEM; multiple OSSL_DECODER_from_bio() calls may be needed.
@@ -128,48 +174,18 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass)
      * we use the different selection as a workaround.
      * https://github.com/openssl/openssl/issues/20657
      */
-    OSSL_DECODER_CTX_free(dctx);
-    dctx = NULL;
-    dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL,
-                                         EVP_PKEY_KEYPAIR, NULL, NULL);
-    if (!dctx)
-        goto out;
-    if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
+    pkey = ossl_pkey_read(bio, "PEM", EVP_PKEY_KEYPAIR, pass);
+    if (pkey) {
         goto out;
-    while (1) {
-        if (OSSL_DECODER_from_bio(dctx, bio) == 1)
-            goto out;
-        if (BIO_eof(bio))
-            break;
-        pos2 = BIO_tell(bio);
-        if (pos2 < 0 || pos2 <= pos)
-            break;
-        ossl_clear_error();
-        pos = pos2;
     }
 
-    OSSL_BIO_reset(bio);
-    OSSL_DECODER_CTX_free(dctx);
-    dctx = NULL;
-    dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL, 0, NULL, NULL);
-    if (!dctx)
-        goto out;
-    if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
-        goto out;
-    while (1) {
-        if (OSSL_DECODER_from_bio(dctx, bio) == 1)
+    for (i = 0; i < selection_num; i++) {
+        pkey = ossl_pkey_read(bio, "PEM", selections[i], pass);
+        if (pkey) {
             goto out;
-        if (BIO_eof(bio))
-            break;
-        pos2 = BIO_tell(bio);
-        if (pos2 < 0 || pos2 <= pos)
-            break;
-        ossl_clear_error();
-        pos = pos2;
+        }
     }
-
   out:
-    OSSL_DECODER_CTX_free(dctx);
     return pkey;
 }
 #else