Skip to content

[GR-27173] Automatic conditional configuration generation. #4270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Mar 11, 2022
Merged

[GR-27173] Automatic conditional configuration generation. #4270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
06c2891
Allow the native-image agent to generate conditional configuration.
Jan 28, 2022
aef27a2
Simplifications.
peter-hofer Jan 28, 2022
3cb6015
Add conditional configuration tests
Feb 4, 2022
9761a10
Cleanup
Feb 8, 2022
e2c7dbf
General code cleanup. Introduce new format for human readable configu…
Feb 24, 2022
555d1e6
Add the capability to filter by regex in filter files
Feb 28, 2022
e40f3f8
Cleanup
Mar 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 44 additions & 1 deletion substratevm/mx.substratevm/mx_substratevm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,13 @@ def is_musl_supported():
return False


def build_native_image_agent(native_image):
agentfile = mx_subst.path_substitutions.substitute('<lib:native-image-agent>')
agentname = agentfile.rsplit('.', 1)[0] # remove platform-specific file extension
native_image(['--macro:native-image-agent-library', '-H:Name=' + agentname, '-H:Path=' + svmbuild_dir()])
return svmbuild_dir() + '/' + agentfile


class GraalVMConfig(collections.namedtuple('GraalVMConfig', 'primary_suite_dir, dynamicimports, disable_libpolyglot, force_bash_launchers, skip_libraries, exclude_components, native_images')):
@classmethod
def build(cls, primary_suite_dir=None, dynamicimports=None, disable_libpolyglot=True, force_bash_launchers=True, skip_libraries=True,
Expand Down Expand Up @@ -218,7 +225,8 @@ def __getattr__(self, name):
'benchmarktest',
"nativeimagehelp",
'muslcbuild',
'hellomodule'
'hellomodule',
'condconfig',
])

def vm_native_image_path(config=None):
Expand Down Expand Up @@ -380,6 +388,11 @@ def svm_gate_body(args, tasks):
with native_image_context(IMAGE_ASSERTION_FLAGS) as native_image:
native_unittests_task()

with Task('conditional configuration tests', tasks, tags=[GraalTags.condconfig]) as t:
if t:
with native_image_context(IMAGE_ASSERTION_FLAGS) as native_image:
conditional_config_task(native_image)

with Task('native unittests quickbuild', tasks, tags=[GraalTags.test_quickbuild]) as t:
if t:
with native_image_context(IMAGE_ASSERTION_FLAGS) as native_image:
Expand Down Expand Up @@ -480,6 +493,32 @@ def native_unittests_task(extra_build_args=None):
native_unittest(['--builder-on-modulepath', '--build-args', _native_unittest_features] + additional_build_args)


def conditional_config_task(native_image):
agent_path = build_native_image_agent(native_image)
config_dir = join(svmbuild_dir(), 'cond-config-test-config')
if exists(config_dir):
mx.rmtree(config_dir)
conditional_config_filter_path = join(svmbuild_dir(), 'conditional-config-filter.json')
with open(conditional_config_filter_path, 'w') as conditional_config_filter:
conditional_config_filter.write(
'''
{
"rules": [
{"includeClasses": "com.oracle.svm.configure.test.conditionalconfig.**"}
]
}
'''
)
agent_opts = ['config-output-dir=' + config_dir, 'experimental-conditional-config-filter-file=' + conditional_config_filter_path]
jvm_unittest(['-agentpath:' + agent_path + '=' + ','.join(agent_opts),
'-Dcom.oracle.svm.configure.test.conditionalconfig.ConfigurationGenerator.enabled=true',
'com.oracle.svm.configure.test.conditionalconfig.ConfigurationGenerator'])

jvm_unittest(['-Dcom.oracle.svm.configure.test.conditionalconfig.ConfigurationVerifier.configpath=' + config_dir,
"-Dcom.oracle.svm.configure.test.conditionalconfig.ConfigurationVerifier.enabled=true",
'com.oracle.svm.configure.test.conditionalconfig.ConfigurationVerifier'])


def javac_image_command(javac_path):
return [join(javac_path, 'javac'), '-proc:none'] + (
# We need to set java.home as com.sun.tools.javac.file.Locations.<clinit> can't handle `null`.
Expand Down Expand Up @@ -572,6 +611,10 @@ def _native_unittest(native_image, cmdline_args):
_native_junit(native_image, unittest_args, unmask(pargs.build_args), unmask(pargs.run_args), blacklist, whitelist, pargs.preserve_image, builder_on_modulepath)


def jvm_unittest(args):
return mx_unittest.unittest(['--suite', 'substratevm'] + args)


def js_image_test(jslib, bench_location, name, warmup_iterations, iterations, timeout=None, bin_args=None):
bin_args = bin_args if bin_args is not None else []
jsruncmd = [get_js_launcher(jslib)] + bin_args + [join(bench_location, 'harness.js'), '--', join(bench_location, name + '.js'),
Expand Down
94 changes: 65 additions & 29 deletions substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/NativeImageAgent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,19 +54,24 @@
import org.graalvm.nativeimage.ProcessProperties;
import org.graalvm.nativeimage.hosted.Feature;

import com.oracle.svm.agent.conditionalconfig.ConditionalConfigurationWriter;
import com.oracle.svm.agent.configwithorigins.ConfigurationWithOriginsWriter;
import com.oracle.svm.agent.configwithorigins.MethodInfoRecordKeeper;
import com.oracle.svm.agent.ignoredconfig.AgentMetaInfProcessor;
import com.oracle.svm.agent.predicatedconfig.ConfigurationWithOriginsResultWriter;
import com.oracle.svm.agent.predicatedconfig.MethodInfoRecordKeeper;
import com.oracle.svm.agent.stackaccess.EagerlyLoadedJavaStackAccess;
import com.oracle.svm.agent.stackaccess.InterceptedState;
import com.oracle.svm.agent.stackaccess.OnDemandJavaStackAccess;
import com.oracle.svm.agent.tracing.ConfigurationResultWriter;
import com.oracle.svm.agent.tracing.TraceFileWriter;
import com.oracle.svm.agent.tracing.core.Tracer;
import com.oracle.svm.agent.tracing.core.TracingResultWriter;
import com.oracle.svm.configure.config.ConditionalConfigurationPredicate;
import com.oracle.svm.configure.config.ConfigurationFileCollection;
import com.oracle.svm.configure.config.ConfigurationSet;
import com.oracle.svm.configure.filters.ComplexFilter;
import com.oracle.svm.configure.filters.ConfigurationFilter;
import com.oracle.svm.configure.filters.FilterConfigurationParser;
import com.oracle.svm.configure.filters.RuleNode;
import com.oracle.svm.configure.filters.HierarchyFilterNode;
import com.oracle.svm.configure.trace.AccessAdvisor;
import com.oracle.svm.configure.trace.TraceProcessor;
import com.oracle.svm.core.SubstrateUtil;
Expand Down Expand Up @@ -114,8 +119,8 @@ protected JNIHandleSet constructJavaHandles(JNIEnvironment env) {
protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks callbacks, String options) {
String traceOutputFile = null;
String configOutputDir = null;
ConfigurationSet mergeConfigs = new ConfigurationSet();
ConfigurationSet omittedConfigs = new ConfigurationSet();
ConfigurationFileCollection mergeConfigs = new ConfigurationFileCollection();
ConfigurationFileCollection omittedConfigs = new ConfigurationFileCollection();
boolean builtinCallerFilter = true;
boolean builtinHeuristicFilter = true;
List<String> callerFilterFiles = new ArrayList<>();
Expand All @@ -125,6 +130,8 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
boolean experimentalOmitClasspathConfig = false;
boolean build = false;
boolean configurationWithOrigins = false;
List<String> conditionalConfigUserPackageFilterFiles = new ArrayList<>();
List<String> conditionalConfigClassNameFilterFiles = new ArrayList<>();
int configWritePeriod = -1; // in seconds
int configWritePeriodInitialDelay = 1; // in seconds
boolean trackReflectionMetadata = true;
Expand Down Expand Up @@ -196,6 +203,10 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
build = Boolean.parseBoolean(getTokenValue(token));
} else if (token.equals("experimental-configuration-with-origins")) {
configurationWithOrigins = true;
} else if (token.startsWith("experimental-conditional-config-filter-file=")) {
conditionalConfigUserPackageFilterFiles.add(getTokenValue(token));
} else if (token.startsWith("conditional-config-class-filter-file=")) {
conditionalConfigClassNameFilterFiles.add(getTokenValue(token));
} else if (token.equals("track-reflection-metadata")) {
trackReflectionMetadata = true;
} else if (token.startsWith("track-reflection-metadata=")) {
Expand All @@ -210,6 +221,10 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
inform("no output/build options provided, tracking dynamic accesses and writing configuration to directory: " + configOutputDir);
}

if (configurationWithOrigins && !conditionalConfigUserPackageFilterFiles.isEmpty()) {
return error(5, "The agent can only be used in either the configuration with origins mode or the predefined classes mode.");
}

if (configurationWithOrigins && !mergeConfigs.isEmpty()) {
configurationWithOrigins = false;
inform("using configuration with origins with configuration merging is currently unsupported. Disabling configuration with origins mode.");
Expand All @@ -219,30 +234,34 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
warn("using experimental configuration with origins mode. Note that native-image cannot process these files, and this flag may change or be removed without a warning!");
}

RuleNode callerFilter = null;
ComplexFilter callerFilter = null;
HierarchyFilterNode callerFilterHierarchyFilterNode = null;
if (!builtinCallerFilter) {
callerFilter = RuleNode.createRoot();
callerFilter.addOrGetChildren("**", RuleNode.Inclusion.Include);
callerFilterHierarchyFilterNode = HierarchyFilterNode.createInclusiveRoot();
callerFilter = new ComplexFilter(callerFilterHierarchyFilterNode);
}

if (!callerFilterFiles.isEmpty()) {
if (callerFilter == null) {
callerFilter = AccessAdvisor.copyBuiltinCallerFilterTree();
if (callerFilterHierarchyFilterNode == null) {
callerFilterHierarchyFilterNode = AccessAdvisor.copyBuiltinCallerFilterTree();
callerFilter = new ComplexFilter(callerFilterHierarchyFilterNode);
}
if (!parseFilterFiles(callerFilter, callerFilterFiles)) {
return 1;
}
}

RuleNode accessFilter = null;
ComplexFilter accessFilter = null;
if (!accessFilterFiles.isEmpty()) {
accessFilter = AccessAdvisor.copyBuiltinAccessFilterTree();
accessFilter = new ComplexFilter(AccessAdvisor.copyBuiltinAccessFilterTree());
if (!parseFilterFiles(accessFilter, accessFilterFiles)) {
return 1;
}
}

final MethodInfoRecordKeeper recordKeeper = new MethodInfoRecordKeeper(configurationWithOrigins);
final Supplier<InterceptedState> interceptedStateSupplier = configurationWithOrigins ? EagerlyLoadedJavaStackAccess.stackAccessSupplier()
boolean shouldTraceOriginInformation = configurationWithOrigins || !conditionalConfigUserPackageFilterFiles.isEmpty();
final MethodInfoRecordKeeper recordKeeper = new MethodInfoRecordKeeper(shouldTraceOriginInformation);
final Supplier<InterceptedState> interceptedStateSupplier = shouldTraceOriginInformation ? EagerlyLoadedJavaStackAccess.stackAccessSupplier()
: OnDemandJavaStackAccess.stackAccessSupplier();

if (configOutputDir != null) {
Expand Down Expand Up @@ -273,21 +292,39 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
ignoreConfigFromClasspath(jvmti, omittedConfigs);
}
AccessAdvisor advisor = createAccessAdvisor(builtinHeuristicFilter, callerFilter, accessFilter);
TraceProcessor omittedConfigProcessor = null;
TraceProcessor processor = new TraceProcessor(advisor);
ConfigurationSet omittedConfiguration = new ConfigurationSet();
Predicate<String> shouldExcludeClassesWithHash = null;
if (!omittedConfigs.isEmpty()) {
Function<IOException, Exception> ignore = e -> {
warn("Failed to load omitted config: " + e);
return null;
};
omittedConfigProcessor = new TraceProcessor(advisor, omittedConfigs.loadJniConfig(ignore), omittedConfigs.loadReflectConfig(ignore),
omittedConfigs.loadProxyConfig(ignore), omittedConfigs.loadResourceConfig(ignore), omittedConfigs.loadSerializationConfig(ignore),
omittedConfigs.loadPredefinedClassesConfig(null, null, ignore), null);
shouldExcludeClassesWithHash = omittedConfigProcessor.getPredefinedClassesConfiguration()::containsClassWithHash;
omittedConfiguration = omittedConfigs.loadConfigurationSet(ignore, null, null);
shouldExcludeClassesWithHash = omittedConfiguration.getPredefinedClassesConfiguration()::containsClassWithHash;
}

if (configurationWithOrigins) {
ConfigurationWithOriginsResultWriter writer = new ConfigurationWithOriginsResultWriter(advisor, recordKeeper);
ConfigurationWithOriginsWriter writer = new ConfigurationWithOriginsWriter(processor, recordKeeper);
tracer = writer;
tracingResultWriter = writer;
} else if (!conditionalConfigUserPackageFilterFiles.isEmpty()) {
ComplexFilter userCodeFilter = new ComplexFilter(HierarchyFilterNode.createRoot());
if (!parseFilterFiles(userCodeFilter, conditionalConfigUserPackageFilterFiles)) {
return 2;
}
ComplexFilter classNameFilter;
if (!conditionalConfigClassNameFilterFiles.isEmpty()) {
classNameFilter = new ComplexFilter(HierarchyFilterNode.createRoot());
if (!parseFilterFiles(classNameFilter, conditionalConfigClassNameFilterFiles)) {
return 3;
}
} else {
classNameFilter = new ComplexFilter(HierarchyFilterNode.createInclusiveRoot());
}

ConditionalConfigurationPredicate predicate = new ConditionalConfigurationPredicate(classNameFilter);
ConditionalConfigurationWriter writer = new ConditionalConfigurationWriter(processor, recordKeeper, userCodeFilter, predicate);
tracer = writer;
tracingResultWriter = writer;
} else {
Expand All @@ -302,10 +339,9 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
}
return e; // rethrow
};
TraceProcessor processor = new TraceProcessor(advisor, mergeConfigs.loadJniConfig(handler), mergeConfigs.loadReflectConfig(handler),
mergeConfigs.loadProxyConfig(handler), mergeConfigs.loadResourceConfig(handler), mergeConfigs.loadSerializationConfig(handler),
mergeConfigs.loadPredefinedClassesConfig(predefinedClassDestDirs, shouldExcludeClassesWithHash, handler), omittedConfigProcessor);
ConfigurationResultWriter writer = new ConfigurationResultWriter(processor);

ConfigurationSet configuration = mergeConfigs.loadConfigurationSet(handler, predefinedClassDestDirs, shouldExcludeClassesWithHash);
ConfigurationResultWriter writer = new ConfigurationResultWriter(processor, configuration, omittedConfiguration);
tracer = writer;
tracingResultWriter = writer;
}
Expand Down Expand Up @@ -368,7 +404,7 @@ private static <T> T usage(T result, String message) {
return result;
}

private static AccessAdvisor createAccessAdvisor(boolean builtinHeuristicFilter, RuleNode callerFilter, RuleNode accessFilter) {
private static AccessAdvisor createAccessAdvisor(boolean builtinHeuristicFilter, ConfigurationFilter callerFilter, ConfigurationFilter accessFilter) {
AccessAdvisor advisor = new AccessAdvisor();
advisor.setHeuristicsEnabled(builtinHeuristicFilter);
if (callerFilter != null) {
Expand All @@ -388,15 +424,15 @@ private static int parseIntegerOrNegative(String number) {
}
}

private static boolean parseFilterFiles(RuleNode filter, List<String> filterFiles) {
private static boolean parseFilterFiles(ComplexFilter filter, List<String> filterFiles) {
for (String path : filterFiles) {
try {
new FilterConfigurationParser(filter).parseAndRegister(Paths.get(path).toUri());
} catch (Exception e) {
return error(false, "cannot parse filter file " + path + ": " + e);
}
}
filter.removeRedundantNodes();
filter.getHierarchyFilterNode().removeRedundantNodes();
return true;
}

Expand All @@ -422,7 +458,7 @@ private void setupExecutorServiceForPeriodicConfigurationCapture(int writePeriod
initialDelay, writePeriod, TimeUnit.SECONDS);
}

private static void ignoreConfigFromClasspath(JvmtiEnv jvmti, ConfigurationSet ignoredConfigSet) {
private static void ignoreConfigFromClasspath(JvmtiEnv jvmti, ConfigurationFileCollection ignoredConfigCollection) {
String classpath = Support.getSystemProperty(jvmti, "java.class.path");
String sep = Support.getSystemProperty(jvmti, "path.separator");
if (sep == null) {
Expand All @@ -436,7 +472,7 @@ private static void ignoreConfigFromClasspath(JvmtiEnv jvmti, ConfigurationSet i
}
}

AgentMetaInfProcessor processor = new AgentMetaInfProcessor(ignoredConfigSet);
AgentMetaInfProcessor processor = new AgentMetaInfProcessor(ignoredConfigCollection);
for (String cpEntry : classpath.split(sep)) {
try {
NativeImageMetaInfWalker.walkMetaInfForCPEntry(Paths.get(cpEntry), processor);
Expand Down
Loading