OpenSSL 1.1.1 cert verification port (#239)

an attempt to port over verify_chain (build_chain) and related bits from OpenSSL 1.1.1

JOSSL's current certificate verification algorithm is rather straightforward and dates back to OpenSSL 0.9 days.

Several times we tried porting over newer code to enhance verification (e.g. to consider alternate chains) but failed due the magnitude of the task.

The PR is an attempt for a minimal viable product in terms of OpenSSL 1.1.1 compatible chain verification.
No relevant security features should be missing - if so than they are likely not present in the legacy algorithm as well.

Author: kares

src/main/java/org/jruby/ext/openssl/OCSPBasicResponse.java

@@ -530,6 +530,9 @@ private boolean checkDelegated(X509Cert signerCA) {
         catch (CertificateParsingException e) {
             throw newOCSPError(getRuntime(), e);
         }
+        catch (IOException e) {
+            throw newOCSPError(getRuntime(), e);
+        }
     }
 
     private boolean matchIssuerId(X509Cert signerCA, CertificateID certId, List<SingleResp> singleResponses) throws IOException {

src/main/java/org/jruby/ext/openssl/SSLContext.java

@@ -945,7 +945,7 @@ StoreContext createStoreContext(final String purpose) {
             // for verify_cb
             storeContext.setExtraData(1, store.getExtraData(1));
             if ( purpose != null ) storeContext.setDefault(purpose);
-            storeContext.verifyParameter.inherit(store.verifyParameter);
+            storeContext.getParam().inherit(store.getParam());
             return storeContext;
         }
 
@@ -1109,14 +1109,14 @@ private void verifyChain(final StoreContext storeContext) throws CertificateExce
                 ok = storeContext.verifyCertificate();
             }
             catch (Exception e) {
-                internalContext.setLastVerifyResult(storeContext.error);
-                if ( storeContext.error == X509Utils.V_OK ) {
+                internalContext.setLastVerifyResult(storeContext.getError());
+                if ( storeContext.getError() == X509Utils.V_OK ) {
                     internalContext.setLastVerifyResult(X509Utils.V_ERR_CERT_REJECTED);
                 }
                 throw new CertificateException("certificate verify failed", e);
             }
 
-            internalContext.setLastVerifyResult(storeContext.error);
+            internalContext.setLastVerifyResult(storeContext.getError());
             if ( ok == 0 ) {
                 throw new CertificateException("certificate verify failed");
             }

src/main/java/org/jruby/ext/openssl/x509store/Function1.java

@@ -33,11 +33,5 @@
  * @author <a href="mailto:[email protected]">Ola Bini</a>
  */
 interface Function1<T> {
-    static class Empty implements Function1 {
-        public int call(Object arg0) {
-            return -1;
-        }
-    }
-    public static final Function1.Empty EMPTY = new Empty();
     int call(T arg0) throws Exception;
 }// Function1

src/main/java/org/jruby/ext/openssl/x509store/Function2.java

@@ -33,11 +33,5 @@
  * @author <a href="mailto:[email protected]">Ola Bini</a>
  */
 interface Function2<T, U> {
-    static class Empty implements Function2 {
-        public int call(Object arg0, Object arg1) {
-            return -1;
-        }
-    }
-    public static final Function2.Empty EMPTY = new Empty();
     int call(T arg0, U arg1) throws Exception;
 }// Function2

src/main/java/org/jruby/ext/openssl/x509store/Function3.java

@@ -33,11 +33,5 @@
  * @author <a href="mailto:[email protected]">Ola Bini</a>
  */
 interface Function3<T, U, V> {
-    static class Empty implements Function3 {
-        public int call(Object arg0,Object arg1,Object arg2) {
-            return -1;
-        }
-    }
-    public static final Function3.Empty EMPTY = new Empty();
     int call(T arg0, U arg1, V arg2) throws Exception;
 }// Function3

src/main/java/org/jruby/ext/openssl/x509store/Function4.java

@@ -33,11 +33,5 @@
  * @author <a href="mailto:[email protected]">Ola Bini</a>
  */
 interface Function4<T, U, V, X> {
-    static class Empty implements Function4 {
-        public int call(Object arg0,Object arg1,Object arg2,Object arg3) {
-            return -1;
-        }
-    }
-    public static final Function4.Empty EMPTY = new Empty();
     int call(T arg0, U arg1, V arg2, X arg3) throws Exception;
 }// Function4

src/main/java/org/jruby/ext/openssl/x509store/Function5.java

@@ -33,11 +33,5 @@
  * @author <a href="mailto:[email protected]">Ola Bini</a>
  */
 interface Function5<T, U, V, X, Y> {
-    static class Empty implements Function5 {
-        public int call(Object arg0,Object arg1,Object arg2,Object arg3,Object arg4) {
-            return -1;
-        }
-    }
-    public static final Function5.Empty EMPTY = new Empty();
     int call(T arg0, U arg1, V arg2, X arg3, Y arg4) throws Exception;
 }// Function5

src/main/java/org/jruby/ext/openssl/x509store/Lookup.java

@@ -81,7 +81,7 @@ public Lookup(Ruby runtime, LookupMethod method) {
         this.runtime = runtime;
 
         final LookupMethod.NewItemFunction newItem = method.newItem;
-        if ( newItem != null && newItem != Function1.EMPTY ) {
+        if ( newItem != null ) {
             final int result;
             try {
                 result = newItem.call(this);
@@ -128,7 +128,7 @@ public static LookupMethod fileLookup() {
     public int control(final int cmd, final String argc, final long argl, final String[] ret) throws Exception {
         if ( method == null ) return -1;
 
-        if ( method.control != null && method.control != Function5.EMPTY ) {
+        if ( method.control != null ) {
             return method.control.call(this, Integer.valueOf(cmd), argc, Long.valueOf(argl), ret);
         }
         return 1;
@@ -364,7 +364,7 @@ private String envEntry(final String key) {
      * c: X509_LOOKUP_free
      */
     public void free() throws Exception {
-        if ( method != null && method.free != null && method.free != Function1.EMPTY ) {
+        if ( method != null && method.free != null ) {
             method.free.call(this);
         }
     }
@@ -374,7 +374,7 @@ public void free() throws Exception {
      */
     public int init() throws Exception {
         if ( method == null ) return 0;
-        if ( method.init != null && method.init != Function1.EMPTY ) {
+        if ( method.init != null ) {
             return method.init.call(this);
         }
         return 1;
@@ -384,7 +384,7 @@ public int init() throws Exception {
      * c: X509_LOOKUP_by_subject
      */
     public int bySubject(final int type, final Name name, final X509Object[] ret) throws Exception {
-        if ( method == null || method.getBySubject == null || method.getBySubject == Function4.EMPTY ) {
+        if ( method == null || method.getBySubject == null ) {
             return X509_LU_FAIL;
         }
         if ( skip ) return 0;
@@ -395,7 +395,7 @@ public int bySubject(final int type, final Name name, final X509Object[] ret) th
      * c: X509_LOOKUP_by_issuer_serial
      */
     public int byIssuerSerialNumber(final int type, final Name name, final BigInteger serial, final X509Object[] ret) throws Exception {
-        if ( method == null || method.getByIssuerSerialNumber == null || method.getByIssuerSerialNumber == Function5.EMPTY ) {
+        if ( method == null || method.getByIssuerSerialNumber == null ) {
             return X509_LU_FAIL;
         }
         return method.getByIssuerSerialNumber.call(this, Integer.valueOf(type), name, serial, ret);
@@ -405,7 +405,7 @@ public int byIssuerSerialNumber(final int type, final Name name, final BigIntege
      * c: X509_LOOKUP_by_fingerprint
      */
     public int byFingerprint(final int type, final String bytes, final X509Object[] ret) throws Exception {
-        if ( method == null || method.getByFingerprint == null || method.getByFingerprint == Function4.EMPTY ) {
+        if ( method == null || method.getByFingerprint == null ) {
             return X509_LU_FAIL;
         }
         return method.getByFingerprint.call(this, Integer.valueOf(type), bytes, ret);
@@ -415,7 +415,7 @@ public int byFingerprint(final int type, final String bytes, final X509Object[]
      * c: X509_LOOKUP_by_alias
      */
     public int byAlias(final int type, final String alias, final X509Object[] ret) throws Exception {
-        if ( method == null || method.getByAlias == null || method.getByAlias == Function4.EMPTY ) {
+        if ( method == null || method.getByAlias == null ) {
             return X509_LU_FAIL;
         }
         return method.getByAlias.call(this, Integer.valueOf(type), alias, ret);
@@ -427,7 +427,7 @@ public int byAlias(final int type, final String alias, final X509Object[] ret) t
     public int shutdown() throws Exception {
         if ( method == null ) return 0;
 
-        if ( method.shutdown != null && method.shutdown != Function1.EMPTY ) {
+        if ( method.shutdown != null ) {
             return method.shutdown.call(this);
         }
         return 1;