8364484: misc tests fail with Received fatal alert: handshake_failure

Backport-of: 724e8c076e1aed05de893ef9366af0e62cc2ac2b

Author: GoeLin

test/jdk/javax/management/security/SecurityTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -142,6 +142,9 @@ static void setTrustStoreProperties(Map<String, Object> map) {
      * map  (argName, value) format, then calls original test's run method.
      */
     public static void main(String args[]) throws Exception {
+        // Disable default KeyManager's certificate checking so we can use
+        // a certificate signed with MD5withRSA algorithm.
+        System.setProperty("jdk.tls.SunX509KeyManager.certChecking", "false");
 
         System.out.println("=================================================");
 
@@ -529,6 +532,10 @@ private static class ClientSide {
         private MBeanServerConnection mbsc = null;
 
         public static void main(String args[]) throws Exception {
+            // Disable default KeyManager's certificate checking so we can use
+            // a certificate signed with MD5withRSA algorithm.
+            System.setProperty("jdk.tls.SunX509KeyManager.certChecking",
+                    "false");
 
             // Parses parameters
             Utils.parseDebugProperties();