Fixes for serialization reflection registration

Author: vjovanov

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reflect/serialize/SerializationFeature.java

@@ -34,6 +34,7 @@
 import java.lang.invoke.SerializedLambda;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Member;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
@@ -204,12 +205,8 @@ private static void registerLambdasFromConstantNodesInGraph(StructuredGraph grap
             Class<?> lambdaClass = getLambdaClassFromConstantNode(cNode);
 
             if (lambdaClass != null && Serializable.class.isAssignableFrom(lambdaClass)) {
-                try {
-                    Method serializeLambdaMethod = lambdaClass.getDeclaredMethod("writeReplace");
-                    RuntimeReflection.register(serializeLambdaMethod);
-                } catch (NoSuchMethodException e) {
-                    throw VMError.shouldNotReachHere("Serializable lambda class must contain the writeReplace method.");
-                }
+                RuntimeReflection.register(ReflectionUtil.lookupMethod(lambdaClass, "writeReplace"));
+                SerializationBuilder.registerSerializationUIDElements(lambdaClass, false);
             }
         }
     }
@@ -345,15 +342,20 @@ final class SerializationBuilder extends ConditionalConfigurationRegistry implem
     private final SerializationDenyRegistry denyRegistry;
     private final ConfigurationTypeResolver typeResolver;
     private final FeatureImpl.DuringSetupAccessImpl access;
+    private final Method disableSerialConstructorChecks;
+    private final Method superHasAccessibleConstructor;
     private boolean sealed;
     private final ProxyRegistry proxyRegistry;
 
     SerializationBuilder(SerializationDenyRegistry serializationDenyRegistry, FeatureImpl.DuringSetupAccessImpl access, ConfigurationTypeResolver typeResolver, ProxyRegistry proxyRegistry) {
         this.access = access;
         Class<?> classDataSlotClazz = access.findClassByName("java.io.ObjectStreamClass$ClassDataSlot");
-        descField = ReflectionUtil.lookupField(classDataSlotClazz, "desc");
-        getDataLayoutMethod = ReflectionUtil.lookupMethod(ObjectStreamClass.class, "getClassDataLayout");
-        stubConstructor = newConstructorForSerialization(SerializationSupport.StubForAbstractClass.class, null);
+        this.descField = ReflectionUtil.lookupField(classDataSlotClazz, "desc");
+        this.getDataLayoutMethod = ReflectionUtil.lookupMethod(ObjectStreamClass.class, "getClassDataLayout");
+        this.stubConstructor = newConstructorForSerialization(SerializationSupport.StubForAbstractClass.class, null);
+        this.disableSerialConstructorChecks = ReflectionUtil.lookupMethod(true, ReflectionFactory.class, "disableSerialConstructorChecks");
+        this.superHasAccessibleConstructor = ReflectionUtil.lookupMethod(ReflectionFactory.class, "superHasAccessibleConstructor", Class.class);
+
         this.denyRegistry = serializationDenyRegistry;
         this.typeResolver = typeResolver;
         this.proxyRegistry = proxyRegistry;
@@ -517,13 +519,41 @@ public void registerWithTargetConstructorClass(ConfigurationCondition condition,
                 Optional.ofNullable(addConstructorAccessor(serializationTargetClass, customTargetConstructorClass))
                                 .map(ReflectionUtil::lookupConstructor)
                                 .ifPresent(RuntimeReflection::register);
+                Class<?> superclass = serializationTargetClass.getSuperclass();
+                if (superclass != null) {
+                    RuntimeReflection.registerAllDeclaredConstructors(superclass);
+                    RuntimeReflection.registerMethodLookup(superclass, "writeReplace");
+                    RuntimeReflection.registerMethodLookup(superclass, "readResolve");
+                }
                 registerForSerialization(serializationTargetClass);
                 registerForDeserialization(serializationTargetClass);
             });
         }
     }
 
-    private static void registerForSerialization(Class<?> serializationTargetClass) {
+    private static void registerQueriesForInheritableMethod(Class<?> clazz, String methodName, Class<?>... args) {
+        Class<?> iter = clazz;
+        while (iter != null) {
+            RuntimeReflection.registerMethodLookup(iter, methodName, args);
+            Method method = ReflectionUtil.lookupMethod(true, clazz, methodName, args);
+            if (method != null) {
+                RuntimeReflection.register(method);
+                break;
+            }
+            iter = iter.getSuperclass();
+        }
+    }
+
+    private static void registerMethod(Class<?> clazz, String methodName, Class<?>... args) {
+        Method method = ReflectionUtil.lookupMethod(true, clazz, methodName, args);
+        if (method != null) {
+            RuntimeReflection.register(method);
+        } else {
+            RuntimeReflection.registerMethodLookup(clazz, methodName, args);
+        }
+    }
+
+    private void registerForSerialization(Class<?> serializationTargetClass) {
 
         /* Proxy classes have special treatment so no registration needed */
         if (Serializable.class.isAssignableFrom(serializationTargetClass) && !Proxy.isProxyClass(serializationTargetClass)) {
@@ -532,15 +562,62 @@ private static void registerForSerialization(Class<?> serializationTargetClass)
              * serialization class consistency, so need to register all constructors, methods and
              * fields.
              */
+            registerSerializationUIDElements(serializationTargetClass, true);
+
+            /*
+             * Required by jdk.internal.reflect.ReflectionFactory.newConstructorForSerialization
+             */
+            Class<?> initCl = serializationTargetClass;
+            boolean initClValid = true;
+            while (Serializable.class.isAssignableFrom(initCl)) {
+                Class<?> prev = initCl;
+                RuntimeReflection.registerAllDeclaredConstructors(initCl);
+                try {
+                    if ((initCl = initCl.getSuperclass()) == null ||
+                                    (!(boolean) disableSerialConstructorChecks.invoke(null) &&
+                                                    !prev.isArray() &&
+                                                    !(Boolean) superHasAccessibleConstructor.invoke(ReflectionFactory.getReflectionFactory(), prev))) {
+                        initClValid = false;
+                        break;
+                    }
+                } catch (InvocationTargetException | IllegalAccessException e) {
+                    throw VMError.shouldNotReachHere(e);
+                }
+            }
+
+            if (initClValid) {
+                RuntimeReflection.registerAllDeclaredConstructors(initCl);
+            }
+
+            Class<?> iter = serializationTargetClass;
+            while (iter != null) {
+                Arrays.stream(iter.getDeclaredFields()).map(Field::getType).forEach(type -> {
+                    RuntimeReflection.registerAllDeclaredMethods(type);
+                    RuntimeReflection.registerAllDeclaredFields(type);
+                    RuntimeReflection.registerAllDeclaredConstructors(type);
+                });
+                iter = iter.getSuperclass();
+            }
+        }
+
+        registerQueriesForInheritableMethod(serializationTargetClass, "writeReplace");
+        registerQueriesForInheritableMethod(serializationTargetClass, "readResolve");
+        registerMethod(serializationTargetClass, "writeObject", ObjectOutputStream.class);
+        registerMethod(serializationTargetClass, "readObjectNoData");
+        registerMethod(serializationTargetClass, "readObject", ObjectInputStream.class);
+    }
+
+    static void registerSerializationUIDElements(Class<?> serializationTargetClass, boolean fullyRegister) {
+        RuntimeReflection.registerAllDeclaredConstructors(serializationTargetClass);
+        RuntimeReflection.registerAllDeclaredMethods(serializationTargetClass);
+        RuntimeReflection.registerAllDeclaredFields(serializationTargetClass);
+        if (fullyRegister) {
+            /* This is here a legacy that we can't remove as it is a breaking change */
             RuntimeReflection.register(serializationTargetClass.getDeclaredConstructors());
             RuntimeReflection.register(serializationTargetClass.getDeclaredMethods());
             RuntimeReflection.register(serializationTargetClass.getDeclaredFields());
         }
-
-        Optional.ofNullable(ReflectionUtil.lookupMethod(true, serializationTargetClass, "writeReplace"))
-                        .ifPresent(RuntimeReflection::register);
-        Optional.ofNullable(ReflectionUtil.lookupMethod(true, serializationTargetClass, "writeObject", ObjectOutputStream.class))
-                        .ifPresent(RuntimeReflection::register);
+        RuntimeReflection.registerFieldLookup(serializationTargetClass, "serialPersistentFields");
     }
 
     public void afterAnalysis() {
@@ -561,14 +638,11 @@ private static void registerForDeserialization(Class<?> serializationTargetClass
             RuntimeReflection.registerAllRecordComponents(serializationTargetClass);
             RuntimeReflection.register(RecordUtils.getRecordComponentAccessorMethods(serializationTargetClass));
         } else if (Externalizable.class.isAssignableFrom(serializationTargetClass)) {
-            Optional.ofNullable(ReflectionUtil.lookupConstructor(true, serializationTargetClass, (Class<?>[]) null))
-                            .ifPresent(RuntimeReflection::register);
+            RuntimeReflection.registerConstructorLookup(serializationTargetClass);
         }
 
-        Optional.ofNullable(ReflectionUtil.lookupMethod(true, serializationTargetClass, "readObject", ObjectInputStream.class))
-                        .ifPresent(RuntimeReflection::register);
-        Optional.ofNullable(ReflectionUtil.lookupMethod(true, serializationTargetClass, "readResolve"))
-                        .ifPresent(RuntimeReflection::register);
+        registerMethod(serializationTargetClass, "readObject", ObjectInputStream.class);
+        registerMethod(serializationTargetClass, "readResolve");
     }
 
     private static Constructor<?> newConstructorForSerialization(Class<?> serializationTargetClass, Constructor<?> customConstructorToCall) {