Merge branch 'master' of https://github.com/oracle/graal into pr-windows-typeinfo

Author: stooke

.github/workflows/main.yml

@@ -56,16 +56,16 @@ jobs:
         include:
           # /compiler
           - env:
-              JDK_VERSION: "19"
+              JDK_VERSION: "20"
               GATE_TAGS: "style,fullbuild,test"
               PRIMARY: "compiler"
           - env:
-              JDK_VERSION: "19"
+              JDK_VERSION: "20"
               GATE_TAGS: "build,bootstraplite"
               PRIMARY: "compiler"
           # /espresso
           - env:
-              JDK_VERSION: "17" # espresso does not support JDK19 yet
+              JDK_VERSION: "17" # espresso does not support JDK20 yet
               GATE_TAGS: "style,fullbuild"
               PRIMARY: "espresso"
           # /substratevm
@@ -96,7 +96,7 @@ jobs:
               PRIMARY: "sulong"
           # /truffle
           - env:
-              JDK_VERSION: "19"
+              JDK_VERSION: "20"
               GATE_TAGS: "" # Truffle does not use tags
               PRIMARY: "truffle"
           # /vm

ci.jsonnet

@@ -1,5 +1,5 @@
 # Common
-local common = import 'common.jsonnet';
+local common = import 'ci/ci_common/common.jsonnet';
 local graal_common = import 'graal-common.json';
 
 # Compiler
@@ -35,12 +35,12 @@ local javadoc = import "ci_includes/publish-javadoc.jsonnet";
 # VM
 local vm = import 'vm/ci/ci_includes/vm.jsonnet';
 
-local verify_ci = (import 'ci-check.libsonnet').verify_ci;
+local verify_ci = (import 'ci/ci_common/ci-check.libsonnet').verify_ci;
 
 {
   # Ensure that entries in common.jsonnet can be resolved.
-  _checkCommon: (import 'common.jsonnet'),
-  ci_resources:: (import 'ci-resources.libsonnet'),
+  _checkCommon: (import 'ci/ci_common/common.jsonnet'),
+  ci_resources:: (import 'ci/ci_common/ci-resources.libsonnet'),
   overlay: graal_common.ci.overlay,
   specVersion: "3",
   builds: [common.add_excludes_guard(b) for b in (

bench-common.libsonnet renamed to ci/ci_common/bench-common.libsonnet

@@ -1,5 +1,5 @@
 {
-  local common = import "common.jsonnet",
+  local common = import "ci/ci_common/common.jsonnet",
   local utils = import "common-utils.libsonnet",
 
   # benchmark job base with automatically generated name
@@ -86,19 +86,20 @@
       ["wait"]
     ],
 
+  // building block used to generate fork builds
   many_forks_benchmarking:: common.build_base + {
-    // building block used to generate fork builds
-    local benchmarking_config_repo = self.ci_resources.infra.benchmarking_config_repo,
+    // assumes that the CI provides the following env vars: CURRENT_BRANCH, BUILD_DIR (as absolute path)
+    local config_repo = "$BUILD_DIR/benchmarking-config",
     environment+: {
-      BENCHMARKING_CONFIG_REPO: "$BUILD_DIR/benchmarking-config",
-      FORK_COUNTS_DIRECTORY: "$BENCHMARKING_CONFIG_REPO/fork-counts",
+      FORK_COUNTS_DIRECTORY: config_repo + "/fork-counts",
       FORK_COUNT_FILE: error "FORK_COUNT_FILE env var must be set to use the many forks execution!"
     },
+    // there is no guarantee that those setup steps run first or from the repo root folder, so all paths must be absolute
     setup+: [
       ["set-export", "CURRENT_BRANCH", ["git", "rev-parse", "--abbrev-ref", "HEAD"]],
       ["echo", "[BENCH-FORKS-CONFIG] Using configuration files from branch ${CURRENT_BRANCH} if it exists remotely."],
-      ["git", "clone", benchmarking_config_repo, "${BENCHMARKING_CONFIG_REPO}"],
-      ["test", "${CURRENT_BRANCH}", "=", "master", "||", "git", "-C", "${BENCHMARKING_CONFIG_REPO}", "checkout", "--track", "origin/${CURRENT_BRANCH}", "||", "echo", "Using default fork counts since there is no branch named '${CURRENT_BRANCH}' in the benchmarking-config repo."]
+      ["git", "clone", self.ci_resources.infra.benchmarking_config_repo, config_repo],
+      ["test", "${CURRENT_BRANCH}", "=", "master", "||", "git", "-C", config_repo, "checkout", "--track", "origin/${CURRENT_BRANCH}", "||", "echo", "Using default fork counts since there is no branch named '${CURRENT_BRANCH}' in the benchmarking-config repo."]
     ]
   },
 

ci-check.libsonnet renamed to ci/ci_common/ci-check.libsonnet

@@ -75,13 +75,16 @@
   hyphenize(a_list)::
     std.join("-", std.filterMap(function(el) el != null, function(el) std.toString(el), a_list)),
 
+  # Pattern for a guard.includes clause that captures all top-level CI files.
+  top_level_ci:: ["*.json", "*.jsonnet", "*.libsonnet", "ci/**"],
+
   # Adds a CI build predicate to `build` if it is a gate such that it is only
-  # run if a non-documentation file in any of `suites` has been updated.
+  # run if a top level CI file or a non-documentation file in any of `suites` has been updated
   add_gate_predicate(build, suites, extra_includes=[], extra_excludes=[])::
     if std.member(build.targets, "gate") then
     build + {
       guard+: {
-        includes+: [ suite + "/**"      for suite in suites ] + extra_includes + ["/*.jsonnet", "/*.libsonnet", "/*.json"],
+        includes+: [ suite + "/**"      for suite in suites ] + extra_includes + $.top_level_ci,
         excludes+: [ suite + "/docs/**" for suite in suites ] + [ "**.md" ] + extra_excludes
       }
     }
@@ -94,4 +97,22 @@
     if objectHas(o, f) then o[f] else default
   ,
 
+  # Makes all properties of an object visible. By default, this function will turn hidden properties to public ones.
+  # Some properties, namely functions, cannot be manifested as json. Trying to do so will result in an error.
+  # Sometimes, e.g., for error reporting or debugging, it is useful to print it nevertheless, ignoring the
+  # functions.
+  make_visible(o, inc_hidden=true)::
+    local objectFields = if inc_hidden then std.objectFieldsAll else std.objectFields;
+    if std.type(o) == "array" then
+      [$.make_visible(e, inc_hidden=inc_hidden) for e in o]
+    else if std.type(o) == "object" then
+      {
+        [key] : $.make_visible(o[key], inc_hidden=inc_hidden)
+        for key in objectFields(o)
+      }
+    else if std.type(o) == "function" then
+      "<function>"
+    else
+      o
+    ,
 }

ci-resources.libsonnet renamed to ci/ci_common/ci-resources.libsonnet

@@ -1,8 +1,8 @@
 local composable = (import "common-utils.libsonnet").composable;
 
-local mx_version = (import "common.json").mx_version;
-local common_json = composable(import "common.json");
-local repo_config = import 'repo-configuration.libsonnet';
+local mx_version = (import "../../common.json").mx_version;
+local common_json = composable(import "../../common.json");
+local repo_config = import '../repo-configuration.libsonnet';
 local jdks = common_json.jdks;
 local deps = common_json.deps;
 local downloads = common_json.downloads;
@@ -51,7 +51,7 @@ local add_jdk_version(name) =
 
   build_base:: {
     // holds location of CI resources that can easily be overwritten in an overlay
-    ci_resources:: (import "ci-resources.libsonnet"),
+    ci_resources:: (import "ci/ci_common/ci-resources.libsonnet"),
   },
 
   // Job frequencies
@@ -128,6 +128,11 @@ local add_jdk_version(name) =
   labsjdk17LLVM::        self["labsjdk-" + repo_config.graalvm_edition + "-17-llvm"],
   labsjdk19LLVM::        self["labsjdk-" + repo_config.graalvm_edition + "-19-llvm"],
 
+  # Only CE exists for JDK 20 until JDK 20 GA.
+  labsjdk20::            self["labsjdk-ce-20"],
+  labsjdk20Debug::       self["labsjdk-ce-20Debug"],
+  labsjdk20LLVM::        self["labsjdk-ce-20-llvm"],
+
 
   // Hardware definitions
   // ********************

common-utils.libsonnet renamed to ci/ci_common/common-utils.libsonnet

@@ -184,7 +184,7 @@ local r      = import "run-spec.libsonnet";
       //
       local _result =
       {
-        "serialgc": {
+        "gc:serialgc": {
           "<all-os>": {
             "<build-config>": {
               "features": "SerialGC"
@@ -197,7 +197,7 @@ local r      = import "run-spec.libsonnet";
             }
           }
         },
-        "serialgc_musl": {
+        "gc:serialgc_libc:musl": {
           "<all-os>": {
             "<build-config>": {
               "features": "SerialGCMusl"
@@ -210,7 +210,7 @@ local r      = import "run-spec.libsonnet";
             }
           }
         },
-        "g1gc": {
+        "gc:g1gc": {
           "<all-os>": {
             "<build-config>": {
               "features": "G1GC"

common.jsonnet renamed to ci/ci_common/common.jsonnet

@@ -3,7 +3,7 @@ local supported_oss_names = r.supported_oss_names;
 local supported_archs_names = r.supported_archs_names;
 local supported_jdks_names = r.supported_jdks_names;
 
-local std_get = (import "../../common-utils.libsonnet").std_get;
+local std_get = (import "../../ci/ci_common/common-utils.libsonnet").std_get;
 {
   //
   local CONFIG_KEY = "<build-config>",
@@ -103,7 +103,7 @@ local std_get = (import "../../common-utils.libsonnet").std_get;
       for field in std.objectFieldsAll(obj)
     ]);
     // create a stable name from features
-    local variant_name_from_features(features) = std.join("_", [features[f] for f in feature_order if std.member(std.objectFieldsAll(features), f)]);
+    local variant_name_from_features(features) = std.join("_", ["%s:%s" % [f, features[f]] for f in feature_order if std.member(std.objectFieldsAll(features), f)]);
     // turn the features list into a variants dictionary
     local features_to_variant(arr) = [
       {

ci/ci_common/run-spec-examples.libsonnet

@@ -3,7 +3,7 @@ local _impl = import "run-spec-impl.libsonnet";
 local supported_oss_names = r.supported_oss_names;
 local supported_archs_names = r.supported_archs_names;
 local supported_jdks_names = r.supported_jdks_names;
-local std_get = (import "../../common-utils.libsonnet").std_get;
+local std_get = (import "../../ci/ci_common/common-utils.libsonnet").std_get;
 {
   local map_os_arch_jdk(fn) =
     [fn(os, arch, jdk)
@@ -20,7 +20,7 @@ local std_get = (import "../../common-utils.libsonnet").std_get;
     ]
   ,
   // return a table of the target, convert to CSV using `jq -r ".[] | @csv"`
-  target_table(processed_spec, with_variants=false)::
+  target_table(processed_spec, with_variants=false, with_empty=false, task_details_title="task details", task_details_factory=function(spec) "")::
     local map_matrix = map_jdk_os_arch;
     local task_dict = processed_spec.after_pushdown(inc_hidden=true);
     local platform_titles = map_matrix(function(os, arch, jdk) std.join("-", [os, arch, jdk]));
@@ -33,21 +33,51 @@ local std_get = (import "../../common-utils.libsonnet").std_get;
       else
         ""
     ;
-    local get_gate_tags(spec) =
-      std.join(" ", std_get(spec, "mxgate_tags", ["<unavailable>"]))
-    ;
     local cols = std.flattenArrays([
       local default_config = r.get_task_spec(task_dict[build]);
       local as_variants = {
         "": _impl.get_platform_spec(task_dict[build])
       } + if with_variants then std.prune(std_get(_impl.get_platform_spec(task_dict[build]), "variants", default={})) else {};
-      [
+      std.foldl(function(acc, variant) acc +
         local run_spec = as_variants[variant];
-        [build, variant, get_gate_tags(default_config)] + map_matrix(function(os, arch, jdk) get_target(default_config, run_spec, os, arch, jdk))
-        for variant in std.objectFieldsAll(as_variants)
-      ]
+        local matrix = map_matrix(function(os, arch, jdk) get_target(default_config, run_spec, os, arch, jdk));
+        local is_empty = std.set(matrix) == [""];
+        if with_empty || !is_empty then
+          [[build, variant, task_details_factory(default_config)] + matrix]
+        else
+          []
+        ,
+        std.objectFieldsAll(as_variants),
+        [])
       for build in std.objectFieldsAll(task_dict)
     ]);
-    local table = [ ["name", "variant", "gate tags"] + platform_titles] + cols;
+    local table = [ ["name", "variant", task_details_title] + platform_titles] + cols;
     table
+  ,
+  // Removes the 'timelimit' property from an object.
+  // Usually, this is used to remove hard-coded (default) timelimits defined in `common.json`.
+  // These definitions assume that the os/arch definition comes first and will be refined later.
+  // With run-spec, however, this is not true in general because the os/arch is only fixed later
+  // in the pipeline. Thus, hard-coded timelimits would override any previous settings. To resolve
+  // this, we delete the default value altogether and explicitly set the timelimits for all jobs.
+  //
+  // Implementation note: we cannot set the value to `null` and use `std.prune` because that deletes hidden fields.
+  delete_timelimit(b)::
+    local public_fields = std.objectFields(b);
+    std.foldl(function(acc, k) acc +
+      local value = b[k];
+      if std.member(public_fields, k) then
+        if std.type(value) == "string" then
+          { [k]: value }
+        else
+          { [k]+: value }
+      else
+        if std.type(value) == "string" then
+          { [k]:: value }
+        else
+          { [k]+:: value }
+      ,
+      [k for k in std.objectFieldsAll(b) if k != "timelimit"],
+      {}
+    ),
 }