Merge with master.

Author: christianhaeubl

ci/ci_common/bench-common.libsonnet

@@ -20,6 +20,8 @@
       "mount_modules": true
     },
     should_use_hwloc:: std.objectHasAll(self, "is_numa") && self.is_numa && std.length(std.find("bench", self.targets)) > 0,
+    restricted_archs:: null,  # null means all architectures are supported, otherwise it must be a list of allowed archs as strings
+    is_arch_supported(arch):: if self.restricted_archs == null then true else std.member(self.restricted_archs, arch),
     min_jdk_version:: null,
     max_jdk_version:: null,
     is_jdk_supported(jdk_version)::

ci/common.jsonnet

@@ -1,6 +1,6 @@
 # This file is shared between many repositories.
 # All objects defined here are mixins, so you can use them like:
-# myjob: common.linux_amd64 + common.jdks.labsjdk17ce + common.deps.sulong + ...
+# { name: "myjob" } + common.linux_amd64 + common.jdks.labsjdk17ce + common.deps.sulong + ...
 # Note that using a os-arch mixin like linux_amd64 mixin is required for using common.deps.
 
 local common_json = import "../common.json";
@@ -57,6 +57,8 @@ local common_json = import "../common.json";
     "linux-jdk20": { packages+: { "devkit:gcc11.2.0-OL6.4+1": "==0" }},
   },
 
+  # Dependencies
+  # ************
   deps: {
     eclipse: {
       downloads+: {
@@ -161,26 +163,18 @@ local common_json = import "../common.json";
     },
   },
 
-  local deps_linux = {
-    packages+: {
-      git: ">=1.8.3",
-    },
-  },
-  local deps_darwin = {
-  },
-  local deps_windows = {
-  },
-
-  local catch_files = [
-    # Keep in sync with org.graalvm.compiler.debug.StandardPathUtilitiesProvider#DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP
-    "Graal diagnostic output saved in '(?P<filename>[^']+)'",
-    # Keep in sync with org.graalvm.compiler.debug.DebugContext#DUMP_FILE_MESSAGE_REGEXP
-    "Dumping debug output to '(?P<filename>[^']+)'",
-    # Keep in sync with com.oracle.svm.hosted.NativeImageOptions#DEFAULT_ERROR_FILE_NAME
-    " (?P<filename>.+/svm_err_b_\\d+T\\d+\\.\\d+_pid\\d+\\.md)",
-  ],
-
-  # Included in common
+  # Hardware definitions and common fields
+  # **************************************
+  # Note that only platforms (os-arch) are exposed (not os and arch separately),
+  # because this is the simplest way to ensure correct usage and dependencies (e.g. ol7 in linux_amd64).
+  #
+  # To add extra "common" fields for your CI:
+  # * If you already have platforms objects, you could extend them like:
+  #   linux_amd64: common.linux_amd64 + self.my_common,
+  # * Otherwise, just include your common object as well as one of the os-arch objects below in each job:
+  #   { name: "myjob" } + common.linux_amd64 + self.my_common + ...
+  #
+  # This also means self.my_common should no longer include mx, etc as it is already included by the os-arch objects.
   local mx = {
     environment+: {
       MX_PYTHON: "python3.8",
@@ -193,20 +187,15 @@ local common_json = import "../common.json";
     python_version: "3", # To use the correct virtualenv
   },
 
-  # Hardware definitions
-  # ********************
-  # Note that only platforms (os-arch) are exposed (not os and arch separately),
-  # because this is the simplest way to ensure correct usage and dependencies (e.g. ol7 in linux_amd64).
-  # If you want to add extra "common" things for your CI, just define your own objects like:
-  # linux_amd64: common.linux_amd64 + extras,
   local common = mx + {
-    local where = if std.objectHas(self, "name") then " in " + self.name else "",
-    # enforce self.os (useful for generating job names)
-    os:: error "self.os not set" + where,
-    # enforce self.arch (useful for generating job names)
-    arch:: error "self.arch not set" + where,
-    capabilities+: [],
-    catch_files+: catch_files,
+    catch_files+: [
+      # Keep in sync with org.graalvm.compiler.debug.StandardPathUtilitiesProvider#DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP
+      "Graal diagnostic output saved in '(?P<filename>[^']+)'",
+      # Keep in sync with org.graalvm.compiler.debug.DebugContext#DUMP_FILE_MESSAGE_REGEXP
+      "Dumping debug output to '(?P<filename>[^']+)'",
+      # Keep in sync with com.oracle.svm.hosted.NativeImageOptions#DEFAULT_ERROR_FILE_NAME
+      " (?P<filename>.+/svm_err_b_\\d+T\\d+\\.\\d+_pid\\d+\\.md)",
+    ],
   },
 
   local ol7 = {
@@ -215,6 +204,12 @@ local common_json = import "../common.json";
       mount_modules: true,
     },
   },
+  local deps_linux = {
+  },
+  local deps_darwin = {
+  },
+  local deps_windows = {
+  },
 
   local linux   = deps_linux   + common + { os:: "linux",   capabilities+: [self.os] },
   local darwin  = deps_darwin  + common + { os:: "darwin",  capabilities+: [self.os] },

common.json

@@ -11,12 +11,12 @@
     "oraclejdk11":        {"name": "jpg-jdk",   "version": "11.0.11", "build_id": "9", "release": true, "platformspecific": true, "extrabundles": ["static-libs"] },
 
     "oraclejdk17":        {"name": "jpg-jdk",   "version": "17.0.1",  "build_id": "12", "release": true, "platformspecific": true, "extrabundles": ["static-libs"]},
-    "labsjdk-ee-17":      {"name": "labsjdk",   "version": "ee-17.0.6+9-jvmci-23.0-b07", "platformspecific": true },
-    "labsjdk-ee-17Debug": {"name": "labsjdk",   "version": "ee-17.0.6+9-jvmci-23.0-b07-debug", "platformspecific": true },
-    "labsjdk-ee-17-llvm": {"name": "labsjdk",   "version": "ee-17.0.6+9-jvmci-23.0-b07-sulong", "platformspecific": true },
-    "labsjdk-ce-17":      {"name": "labsjdk",   "version": "ce-17.0.6+10-jvmci-23.0-b07", "platformspecific": true },
-    "labsjdk-ce-17Debug": {"name": "labsjdk",   "version": "ce-17.0.6+10-jvmci-23.0-b07-debug", "platformspecific": true },
-    "labsjdk-ce-17-llvm": {"name": "labsjdk",   "version": "ce-17.0.6+10-jvmci-23.0-b07-sulong", "platformspecific": true },
+    "labsjdk-ce-17":      {"name": "labsjdk",   "version": "ce-17.0.7+2-jvmci-23.0-b08", "platformspecific": true },
+    "labsjdk-ce-17Debug": {"name": "labsjdk",   "version": "ce-17.0.7+2-jvmci-23.0-b08-debug", "platformspecific": true },
+    "labsjdk-ce-17-llvm": {"name": "labsjdk",   "version": "ce-17.0.7+2-jvmci-23.0-b08-sulong", "platformspecific": true },
+    "labsjdk-ee-17":      {"name": "labsjdk",   "version": "ee-17.0.7+5-jvmci-23.0-b08", "platformspecific": true },
+    "labsjdk-ee-17Debug": {"name": "labsjdk",   "version": "ee-17.0.7+5-jvmci-23.0-b08-debug", "platformspecific": true },
+    "labsjdk-ee-17-llvm": {"name": "labsjdk",   "version": "ee-17.0.7+5-jvmci-23.0-b08-sulong", "platformspecific": true },
 
     "oraclejdk19":        {"name": "jpg-jdk",   "version": "19",      "build_id": "26", "release": true, "platformspecific": true, "extrabundles": ["static-libs"]},
     "labsjdk-ce-19":      {"name": "labsjdk",   "version": "ce-19.0.1+10-jvmci-23.0-b04", "platformspecific": true },
@@ -27,12 +27,12 @@
     "labsjdk-ee-19-llvm": {"name": "labsjdk",   "version": "ee-19.0.2+7-jvmci-23.0-b05-sulong", "platformspecific": true },
 
     "oraclejdk20":        {"name": "jpg-jdk",   "version": "20",      "build_id": "24", "release": true, "platformspecific": true, "extrabundles": ["static-libs"]},
-    "labsjdk-ce-20":      {"name": "labsjdk",   "version": "ce-20+34-jvmci-23.0-b07", "platformspecific": true },
-    "labsjdk-ce-20Debug": {"name": "labsjdk",   "version": "ce-20+34-jvmci-23.0-b07-debug", "platformspecific": true },
-    "labsjdk-ce-20-llvm": {"name": "labsjdk",   "version": "ce-20+34-jvmci-23.0-b07-sulong", "platformspecific": true },
-    "labsjdk-ee-20":      {"name": "labsjdk",   "version": "ee-20.0.1+4-jvmci-23.0-b07", "platformspecific": true },
-    "labsjdk-ee-20Debug": {"name": "labsjdk",   "version": "ee-20.0.1+4-jvmci-23.0-b07-debug", "platformspecific": true },
-    "labsjdk-ee-20-llvm": {"name": "labsjdk",   "version": "ee-20.0.1+4-jvmci-23.0-b07-sulong", "platformspecific": true }
+    "labsjdk-ce-20":      {"name": "labsjdk",   "version": "ce-20+34-jvmci-23.0-b08", "platformspecific": true },
+    "labsjdk-ce-20Debug": {"name": "labsjdk",   "version": "ce-20+34-jvmci-23.0-b08-debug", "platformspecific": true },
+    "labsjdk-ce-20-llvm": {"name": "labsjdk",   "version": "ce-20+34-jvmci-23.0-b08-sulong", "platformspecific": true },
+    "labsjdk-ee-20":      {"name": "labsjdk",   "version": "ee-20.0.1+6-jvmci-23.0-b08", "platformspecific": true },
+    "labsjdk-ee-20Debug": {"name": "labsjdk",   "version": "ee-20.0.1+6-jvmci-23.0-b08-debug", "platformspecific": true },
+    "labsjdk-ee-20-llvm": {"name": "labsjdk",   "version": "ee-20.0.1+6-jvmci-23.0-b08-sulong", "platformspecific": true }
   },
 
   "eclipse": {

compiler/ci/ci_common/benchmark-builders.jsonnet

@@ -8,41 +8,23 @@
   local main_builds = std.flattenArrays([
     [
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.dacapo + { unicorn_pull_request_benchmarking:: {name: 'libgraal', metrics: ['time']}},
-    c.daily      + hw.x52 + jdk + cc.jargraal + bench.dacapo,
     c.weekly     + hw.x52 + jdk + cc.libgraal + bench.dacapo_size_variants,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.dacapo_size_variants,
     c.weekly     + hw.x52 + jdk + cc.libgraal + bench.dacapo_timing,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.dacapo_timing,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.scala_dacapo,
-    c.daily      + hw.x52 + jdk + cc.jargraal + bench.scala_dacapo,
     c.weekly     + hw.x52 + jdk + cc.libgraal + bench.scala_dacapo_size_variants,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.scala_dacapo_size_variants,
     c.weekly     + hw.x52 + jdk + cc.libgraal + bench.scala_dacapo_timing,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.scala_dacapo_timing,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.renaissance + {unicorn_pull_request_benchmarking:: 'libgraal'},
-    c.daily      + hw.x52 + jdk + cc.jargraal + bench.renaissance,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.specjvm2008,
-    c.daily      + hw.x52 + jdk + cc.jargraal + bench.specjvm2008,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.specjbb2015,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.specjbb2015,
     c.weekly     + hw.x52 + jdk + cc.libgraal + bench.specjbb2015_full_machine,
-    c.monthly    + hw.x52 + jdk + cc.jargraal + bench.specjbb2015_full_machine,
     c.weekly     + hw.x52 + jdk + cc.libgraal + bench.renaissance_0_11,
-    c.monthly    + hw.x52 + jdk + cc.jargraal + bench.renaissance_0_11,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.awfy + {unicorn_pull_request_benchmarking:: 'libgraal'},
-    c.daily      + hw.x52 + jdk + cc.jargraal + bench.awfy,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.microservice_benchmarks,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.microservice_benchmarks,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.renaissance_legacy,
-    c.daily      + hw.x52 + jdk + cc.jargraal + bench.renaissance_legacy,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.micros_graal_whitebox,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.micros_graal_whitebox,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.micros_graal_dist,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.micros_graal_dist,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.micros_misc_graal_dist,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.micros_misc_graal_dist,
     c.daily      + hw.x52 + jdk + cc.libgraal + bench.micros_shootout_graal_dist,
-    c.weekly     + hw.x52 + jdk + cc.jargraal + bench.micros_shootout_graal_dist,
     ]
   for jdk in cc.bench_jdks
   ]),
@@ -57,22 +39,23 @@
   for suite in bench.groups.profiled_suites
   ]),
 
-  // intensive weekly benchmarking
-  local weekly_forks_builds = std.flattenArrays([
-    bc.generate_fork_builds(c.weekly + hw.x52 + jdk + cc.libgraal + suite, subdir='compiler')
+  local weekly_amd64_forks_builds = std.flattenArrays([
+    bc.generate_fork_builds(c.weekly  + hw.x52  + jdk + cc.libgraal + suite, subdir='compiler') +
+    bc.generate_fork_builds(c.monthly + hw.x52  + jdk + cc.jargraal + suite, subdir='compiler')
   for jdk in cc.bench_jdks
   for suite in bench.groups.weekly_forks_suites
   ]),
 
-  local aarch64_builds = std.flattenArrays([
-    [
-    c.weekly + hw.a12c + jdk + cc.libgraal + suite,
-    c.weekly + hw.a12c + jdk + cc.jargraal + suite
-    ]
-  for jdk in cc.bench_jdks
-  for suite in bench.groups.all_suites
+  local weekly_aarch64_forks_builds = std.flattenArrays([
+    bc.generate_fork_builds(c.weekly + hw.a12c + cc.latest_jdk + cc.libgraal + suite, subdir='compiler')
+  for suite in bench.groups.weekly_forks_suites
   ]),
 
+  local aarch64_builds = [
+    c.daily + hw.a12c + cc.latest_jdk + cc.libgraal + suite,
+  for suite in bench.groups.all_suites
+  ],
+
   local avx_builds = [
     c.monthly + hw.x82 + jdk + cc.libgraal + avx + suite,
   for avx in [cc.avx2_mode, cc.avx3_mode]
@@ -93,8 +76,8 @@
   ],
 
 
-  local all_builds = main_builds + weekly_forks_builds + profiling_builds + avx_builds + aarch64_builds + no_tiered_builds + no_profile_info_builds,
-  local filtered_builds = [b for b in all_builds if b.is_jdk_supported(b.jdk_version)],
+  local all_builds = main_builds + weekly_amd64_forks_builds + weekly_aarch64_forks_builds + profiling_builds + avx_builds + aarch64_builds + no_tiered_builds + no_profile_info_builds,
+  local filtered_builds = [b for b in all_builds if b.is_jdk_supported(b.jdk_version) && b.is_arch_supported(b.arch)],
   // adds a "defined_in" field to all builds mentioning the location of this current file
   builds:: [{ defined_in: std.thisFile } + b for b in filtered_builds]
 }

compiler/ci/ci_common/benchmark-suites.libsonnet

@@ -13,10 +13,10 @@
     legacy_and_secondary_suites:: unique_suites([$.renaissance_0_11, $.renaissance_legacy]),
     jmh_micros_suites:: unique_suites([$.micros_graal_dist, $.micros_misc_graal_dist , $.micros_shootout_graal_dist]),
     graal_internals_suites:: unique_suites([$.micros_graal_whitebox]),
-    special_suites:: unique_suites([$.renaissance, $.dacapo_size_variants, $.scala_dacapo_size_variants, $.specjbb2015_full_machine]),
+    special_suites:: unique_suites([$.dacapo_size_variants, $.scala_dacapo_size_variants, $.specjbb2015_full_machine]),
     microservice_suites:: unique_suites([$.microservice_benchmarks]),
 
-    main_suites:: unique_suites([$.specjvm2008] + self.open_suites + self.legacy_and_secondary_suites),
+    main_suites:: unique_suites([$.specjvm2008] + self.open_suites),
     all_suites:: unique_suites(self.main_suites + self.spec_suites + self.jmh_micros_suites + self.special_suites + self.microservice_suites),
 
     weekly_forks_suites:: self.main_suites,
@@ -59,8 +59,8 @@
       self._bench_upload(),
       self.benchmark_cmd + ["dacapo-huge:*", "--"] + self.extra_vm_args
     ],
-    timelimit: "07:00:00",
-    forks_batches:: null, # weekly forks disabled
+    timelimit: "09:00:00",
+    forks_batches:: null,
     forks_timelimit:: null,
     min_jdk_version:: 8,
     max_jdk_version:: null
@@ -266,7 +266,8 @@
       bench_upload
     ],
     timelimit: "7:00:00",
-    min_jdk_version:: 11, # GR-32793: disabled JDK8
+    restricted_archs:: ["amd64"],  # load testers only work on amd64 at the moment: GR-35619
+    min_jdk_version:: 11,
     max_jdk_version:: null
   },
 

compiler/ci/ci_common/compiler-common.libsonnet

@@ -31,9 +31,10 @@
     ]
   },
 
+  latest_jdk:: common["labsjdk-ee-20"],
   bench_jdks:: [
     common["labsjdk-ee-17"],
-    common["labsjdk-ee-20"]
+    self.latest_jdk
   ],
 
   // Benchmarking building blocks

compiler/ci/ci_includes/baseline-benchmarks.jsonnet

@@ -5,33 +5,36 @@
   local bench = (import '../ci_common/benchmark-suites.libsonnet'),
   local hw = bc.bench_hw,
 
-  local hotspot_main_builds = [
+  local hotspot_amd64_builds = [
     c.weekly + hw.x52 + jdk + cc.c2 + suite
   for jdk in cc.bench_jdks
   for suite in bench.groups.all_suites
   ],
 
+  local hotspot_aarch64_builds = [
+    c.weekly + hw.a12c + cc.latest_jdk + cc.c2 + suite
+  for suite in bench.groups.all_suites
+  ],
+
   local hotspot_profiling_builds = std.flattenArrays([
     [
-    c.weekly + hw.x52 + jdk + cc.c2 + cc.enable_profiling   + suite + { job_prefix:: "bench-profiling" },
-    c.weekly + hw.x52 + jdk + cc.c2 + cc.footprint_tracking + suite + { job_prefix:: "bench-footprint" }
+    c.weekly + hw.x52  + cc.latest_jdk + cc.c2 + cc.enable_profiling   + suite + { job_prefix:: "bench-profiling" },
+    c.weekly + hw.a12c + cc.latest_jdk + cc.c2 + cc.enable_profiling   + suite + { job_prefix:: "bench-profiling" },
+    c.weekly + hw.x52  + cc.latest_jdk + cc.c2 + cc.footprint_tracking + suite + { job_prefix:: "bench-footprint" },
+    c.weekly + hw.a12c + cc.latest_jdk + cc.c2 + cc.footprint_tracking + suite + { job_prefix:: "bench-footprint" }
     ]
-  for jdk in cc.bench_jdks
   for suite in bench.groups.profiled_suites
   ]),
 
-  local weekly_forks_builds = std.flattenArrays([
+  local weekly_forks_amd64_builds = std.flattenArrays([
     bc.generate_fork_builds(c.weekly + hw.x52 + jdk + cc.c2 + suite)
   for jdk in cc.bench_jdks
   for suite in bench.groups.weekly_forks_suites
   ]),
 
-  local aarch64_builds = std.flattenArrays([
-    [
-    c.weekly + hw.a12c + jdk + cc.c2 + suite
-    ]
-  for jdk in cc.bench_jdks
-  for suite in bench.groups.all_suites
+  local weekly_forks_aarch64_builds = std.flattenArrays([
+    bc.generate_fork_builds(c.weekly + hw.a12c + cc.latest_jdk + cc.c2 + suite)
+  for suite in bench.groups.weekly_forks_suites
   ]),
 
   local daily_economy_builds = [
@@ -56,8 +59,9 @@
   for suite in bench.groups.main_suites
   ]),
 
-  local all_builds = hotspot_main_builds + hotspot_profiling_builds + weekly_forks_builds + aarch64_builds + daily_economy_builds + weekly_economy_builds + no_tiered_builds,
-  local filtered_builds = [b for b in all_builds if b.is_jdk_supported(b.jdk_version)],
+  local all_builds = hotspot_amd64_builds + hotspot_aarch64_builds + hotspot_profiling_builds +
+    weekly_forks_amd64_builds + weekly_forks_aarch64_builds + daily_economy_builds + weekly_economy_builds + no_tiered_builds,
+  local filtered_builds = [b for b in all_builds if b.is_jdk_supported(b.jdk_version) && b.is_arch_supported(b.arch)],
   // adds a "defined_in" field to all builds mentioning the location of this current file
   builds:: [{ defined_in: std.thisFile } + b for b in filtered_builds]
 }

compiler/mx.compiler/mx_compiler.py

@@ -515,6 +515,11 @@ def compiler_gate_benchmark_runner(tasks, extraVMarguments=None, prefix='', task
             c2BenchVmArgs = [a for a in benchVmArgs if 'JVMCI' not in a]
             _gate_dacapo('fop', 1, ['--jvm-config', 'default'] + c2BenchVmArgs)
 
+    # ensure we can run with --enable-preview
+    with Task(prefix + 'DaCapo_enable-preview:fop', tasks, tags=GraalTags.test, report=task_report_component) as t:
+        if t:
+            _gate_dacapo('fop', 8, ['--enable-preview', '-Dgraal.CompilationFailureAction=ExitVM'])
+
     # run Scala DaCapo benchmarks #
     ###############################
     scala_dacapo_suite = mx_graal_benchmark.ScalaDaCapoBenchmarkSuite()