[GR-38783] Deploy graalvm artifacts to artifact repo

imanemamri · imanemamri · commit 1c47f17971a7 · 2022-11-25T09:56:32.000Z
PullRequest: graal/11935
diff --git a/common.json b/common.json
@@ -1,7 +1,7 @@
 {
   "README": "This file contains definitions that are useful for the hocon and jsonnet CI files of multiple repositories.",
 
-  "mx_version" : "6.11.4",
+  "mx_version" : "6.12.7",
 
   "jdks": {
     "openjdk11":          {"name": "jpg-jdk",   "version": "11.0.11", "build_id": "9", "open": true, "release": true, "platformspecific": true },
diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet
@@ -384,11 +384,25 @@ local devkits = common_json.devkits;
   svm_common_darwin_aarch64:     { environment+: graal_common.svm_deps.common.environment, logs+: graal_common.svm_deps.common.logs} + graal_common.svm_deps.darwin_aarch64,
   svm_common_windows_amd64(jdk): { environment+: graal_common.svm_deps.common.environment, logs+: graal_common.svm_deps.common.logs} + graal_common.svm_deps.windows       + common_json.devkits["windows-jdk" + jdk],
 
-  maven_deploy_sdk: ['--suite', 'sdk', 'maven-deploy', '--validate', 'none', '--all-distribution-types', '--with-suite-revisions-metadata'],
-  maven_deploy_sdk_base:               self.maven_deploy_sdk + ['--tags', 'graalvm',                             vm.binaries_repository],
-  maven_deploy_sdk_base_dry_run:       self.maven_deploy_sdk + ['--tags', 'graalvm',                '--dry-run', vm.binaries_repository],
-  maven_deploy_sdk_components:         self.maven_deploy_sdk + ['--tags', 'installable,standalone',              vm.binaries_repository],
-  maven_deploy_sdk_components_dry_run: self.maven_deploy_sdk + ['--tags', 'installable,standalone', '--dry-run', vm.binaries_repository],
+  maven_deploy_sdk:                     ['--suite', 'sdk', 'maven-deploy', '--validate', 'none', '--all-distribution-types', '--with-suite-revisions-metadata'],
+  deploy_artifacts_sdk(os, base_dist_name=null): (if base_dist_name != null then ['--base-dist-name=' + base_dist_name] else []) + ['--suite', 'sdk', 'deploy-artifacts', '--uploader', if os == 'windows' then 'artifact_uploader.cmd' else 'artifact_uploader'],
+
+  maven_deploy_sdk_base:                    self.maven_deploy_sdk +                     ['--tags', 'graalvm', vm.binaries_repository],
+  artifact_deploy_sdk_base(os, base_dist_name): self.deploy_artifacts_sdk(os, base_dist_name) + ['--tags', 'graalvm'],
+  deploy_sdk_base(os, base_dist_name=null):     [self.mx_vm_common + vm.vm_profiles + self.maven_deploy_sdk_base, self.mx_vm_common + vm.vm_profiles + self.artifact_deploy_sdk_base(os, base_dist_name)],
+
+  maven_deploy_sdk_base_dry_run:                    self.maven_deploy_sdk +                     ['--tags', 'graalvm', '--dry-run', vm.binaries_repository],
+  artifact_deploy_sdk_base_dry_run(os, base_dist_name): self.deploy_artifacts_sdk(os, base_dist_name) + ['--tags', 'graalvm', '--dry-run'],
+  deploy_sdk_base_dry_run(os, base_dist_name=null):     [self.mx_vm_common + vm.vm_profiles + self.maven_deploy_sdk_base_dry_run, self.mx_vm_common + vm.vm_profiles + self.artifact_deploy_sdk_base_dry_run(os, base_dist_name)],
+
+  maven_deploy_sdk_components:    self.maven_deploy_sdk +      ['--tags', 'installable,standalone',              vm.binaries_repository],
+  artifact_deploy_sdk_components(os): self.deploy_artifacts_sdk(os) +  ['--tags', 'installable,standalone',                ],
+  deploy_sdk_components(os):          [$.mx_vm_installables + self.maven_deploy_sdk_components, $.mx_vm_installables + self.artifact_deploy_sdk_components(os)],
+
+  maven_deploy_sdk_components_dry_run:    self.maven_deploy_sdk +     ['--tags', 'installable,standalone', '--dry-run', vm.binaries_repository],
+  artifact_deploy_sdk_components_dry_run(os): self.deploy_artifacts_sdk(os) + ['--tags', 'installable,standalone',                '--dry-run'],
+  deploy_sdk_components_dry_run(os):          [$.mx_vm_installables + self.maven_deploy_sdk_components_dry_run, $.mx_vm_installables + self.artifact_deploy_sdk_components_dry_run(os)],
+
 
   svm_vm_build_ol6_amd64: self.svm_common_linux_amd64 + vm.custom_vm_linux,
 
@@ -412,13 +426,17 @@ local devkits = common_json.devkits;
     },
   },
 
-  linux_deploy: {
+  deploy_build: {
+    deploysArtifacts: true
+  },
+
+  linux_deploy: self.deploy_build + {
     packages+: {
       maven: '>=3.3.9',
     },
   },
 
-  darwin_deploy: self.maven_download_unix + {
+  darwin_deploy: self.deploy_build + self.maven_download_unix + {
     environment+: {
       PATH: '$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH:/usr/local/bin',
     },
@@ -500,13 +518,13 @@ local devkits = common_json.devkits;
     run: $.patch_env('linux', 'amd64', java_version) + [
       $.mx_vm_installables + ['graalvm-show'],
       $.mx_vm_installables + ['build'],
-      $.mx_vm_installables + $.maven_deploy_sdk_components,
+    ] + $.deploy_sdk_components(self.os) + [
       $.mx_vm_installables + $.record_file_sizes,
       $.upload_file_sizes,
     ] + vm.collect_profiles() + $.build_base_graalvm_image + [
       $.mx_vm_common + vm.vm_profiles + $.record_file_sizes,
       $.upload_file_sizes,
-      $.mx_vm_common + vm.vm_profiles + $.maven_deploy_sdk_base,
+    ] + $.deploy_sdk_base(self.os) + [
       ['set-export', 'GRAALVM_HOME', $.mx_vm_common + ['--quiet', '--no-warning', 'graalvm-home']],
     ] + $.create_releaser_notifier_artifact + $.check_base_graalvm_image("linux", "amd64", java_version) + [
       ['set-export', 'GRAALVM_HOME', $.mx_vm_installables + ['--quiet', '--no-warning', 'graalvm-home']],
@@ -519,15 +537,13 @@ local devkits = common_json.devkits;
     run: $.patch_env('linux', 'aarch64', java_version) + [
       $.mx_vm_installables + ['graalvm-show'],
       $.mx_vm_installables + ['build'],
-    ] + [
-      $.mx_vm_installables + $.maven_deploy_sdk_components,
+    ] + $.deploy_sdk_components(self.os) + [
       $.mx_vm_installables + $.record_file_sizes,
       $.upload_file_sizes,
     ] + vm.collect_profiles() + $.build_base_graalvm_image + [
       $.mx_vm_common + vm.vm_profiles + $.record_file_sizes,
       $.upload_file_sizes,
-      $.mx_vm_common + vm.vm_profiles + $.maven_deploy_sdk_base,
-    ] + $.create_releaser_notifier_artifact + $.check_base_graalvm_image("linux", "aarch64", java_version) + [
+    ] + $.deploy_sdk_base(self.os) + $.create_releaser_notifier_artifact + $.check_base_graalvm_image("linux", "aarch64", java_version) + [
       ['set-export', 'GRAALVM_HOME', $.mx_vm_installables + ['--quiet', '--no-warning', 'graalvm-home']],
     ] + vm.check_graalvm_complete_build($.mx_vm_installables, "linux", "aarch64", java_version),
     notify_groups:: ['deploy'],
@@ -537,9 +553,8 @@ local devkits = common_json.devkits;
   deploy_graalvm_base_darwin_amd64(java_version): vm.check_structure + {
     run: $.patch_env('darwin', 'amd64', java_version) + vm.collect_profiles() + $.build_base_graalvm_image + [
       $.mx_vm_common + vm.vm_profiles + $.record_file_sizes,
-      $.upload_file_sizes,
-      $.mx_vm_common + vm.vm_profiles + $.maven_deploy_sdk_base,
-    ] + $.create_releaser_notifier_artifact + $.check_base_graalvm_image("darwin", "amd64", java_version),
+      $.upload_file_sizes
+    ] + $.deploy_sdk_base(self.os) + $.create_releaser_notifier_artifact + $.check_base_graalvm_image("darwin", "amd64", java_version),
     notify_groups:: ['deploy'],
     timelimit: '1:45:00',
   },
@@ -549,7 +564,7 @@ local devkits = common_json.devkits;
       $.mx_vm_installables + ['graalvm-show'],
       $.mx_vm_installables + ['build'],
       ['set-export', 'GRAALVM_HOME', $.mx_vm_installables + ['--quiet', '--no-warning', 'graalvm-home']],
-      $.mx_vm_installables + $.maven_deploy_sdk_components,
+    ] + $.deploy_sdk_components(self.os) + [
       $.mx_vm_installables + $.record_file_sizes,
       $.upload_file_sizes,
     ] + $.create_releaser_notifier_artifact + vm.check_graalvm_complete_build($.mx_vm_installables, "darwin", "amd64", java_version),
@@ -561,8 +576,7 @@ local devkits = common_json.devkits;
     run: $.patch_env('darwin', 'aarch64', java_version) + vm.collect_profiles() + $.build_base_graalvm_image + [
       $.mx_vm_common + vm.vm_profiles + $.record_file_sizes,
       $.upload_file_sizes,
-      $.mx_vm_common + vm.vm_profiles + $.maven_deploy_sdk_base,
-    ] + $.create_releaser_notifier_artifact + $.check_base_graalvm_image("darwin", "aarch64", java_version),
+    ] + $.deploy_sdk_base(self.os) + $.create_releaser_notifier_artifact + $.check_base_graalvm_image("darwin", "aarch64", java_version),
     notify_emails+: ['gilles.m.duboscq@oracle.com', 'bernhard.urban-forster@oracle.com'],
     timelimit: '1:45:00',
   },
@@ -572,7 +586,7 @@ local devkits = common_json.devkits;
       $.mx_vm_installables + ['graalvm-show'],
       $.mx_vm_installables + ['build'],
       ['set-export', 'GRAALVM_HOME', $.mx_vm_installables + ['--quiet', '--no-warning', 'graalvm-home']],
-      $.mx_vm_installables + $.maven_deploy_sdk_components,
+    ] + $.deploy_sdk_components(self.os) + [
       $.mx_vm_installables + $.record_file_sizes,
       $.upload_file_sizes,
     ] +  $.create_releaser_notifier_artifact + vm.check_graalvm_complete_build($.mx_vm_installables, "darwin", "aarch64", java_version),
@@ -584,8 +598,7 @@ local devkits = common_json.devkits;
     run: $.patch_env('windows', 'amd64', java_version) + vm.collect_profiles() + $.build_base_graalvm_image + [
       $.mx_vm_common + vm.vm_profiles + $.record_file_sizes,
       $.upload_file_sizes,
-      $.mx_vm_common + vm.vm_profiles + $.maven_deploy_sdk_base,
-    ] + $.create_releaser_notifier_artifact + $.check_base_graalvm_image("windows", "amd64", java_version),
+    ] + $.deploy_sdk_base(self.os) + $.create_releaser_notifier_artifact + $.check_base_graalvm_image("windows", "amd64", java_version),
     notify_groups:: ['deploy'],
     timelimit: '1:30:00',
   },
@@ -595,7 +608,7 @@ local devkits = common_json.devkits;
       $.mx_vm_installables + ['graalvm-show'],
       $.mx_vm_installables + ['build'],
       ['set-export', 'GRAALVM_HOME', $.mx_vm_installables + ['--quiet', '--no-warning', 'graalvm-home']],
-      $.mx_vm_installables + $.maven_deploy_sdk_components,
+    ] + $.deploy_sdk_components(self.os) + [
       $.mx_vm_installables + $.record_file_sizes,
       $.upload_file_sizes,
     ] + $.create_releaser_notifier_artifact + vm.check_graalvm_complete_build($.mx_vm_installables, "windows", "amd64", java_version),
@@ -606,8 +619,7 @@ local devkits = common_json.devkits;
   deploy_graalvm_ruby(os, arch, java_version): {
     run: vm.collect_profiles() + [
       ['set-export', 'VM_ENV', "${VM_ENV}-ruby"],
-    ] + $.build_base_graalvm_image + [
-      $.mx_vm_common + vm.vm_profiles + $.maven_deploy_sdk_base,
+    ] + $.build_base_graalvm_image + $.deploy_sdk_base(os, 'ruby') + [
       ['set-export', 'GRAALVM_HOME', $.mx_vm_common + ['--quiet', '--no-warning', 'graalvm-home']],
     ] + $.create_releaser_notifier_artifact,
     notify_groups:: ['deploy', 'ruby'],
@@ -640,10 +652,10 @@ local devkits = common_json.devkits;
   deploy_vm_installable_java19_darwin_aarch64: vm.vm_java_19 + self.full_vm_build_darwin_aarch64 + self.darwin_deploy + self.deploy_daily_vm_darwin_aarch64 + self.deploy_graalvm_installables_darwin_aarch64("java19") + {name: 'daily-deploy-vm-installable-java19-darwin-aarch64', diskspace_required: "31GB", notify_groups:: ["deploy"]},
 
   # Windows/AMD64
-  deploy_vm_base_java17_windows_amd64: vm.vm_java_17 + self.svm_common_windows_amd64("17") + self.js_windows_jdk17 + self.deploy_daily_vm_windows_jdk17 + self.deploy_graalvm_base_windows_amd64("java17") + {name: 'daily-deploy-vm-base-java17-windows-amd64', notify_groups:: ["deploy"]},
-  deploy_vm_installable_java17_windows_amd64: vm.vm_java_17 + self.svm_common_windows_amd64("17") + self.js_windows_jdk17 + self.sulong_windows + self.deploy_daily_vm_windows_jdk17 + self.deploy_graalvm_installables_windows_amd64("java17") + {name: 'daily-deploy-vm-installable-java17-windows-amd64', diskspace_required: "31GB", notify_groups:: ["deploy"]},
-  deploy_vm_base_java19_windows_amd64: vm.vm_java_19 + self.svm_common_windows_amd64("19") + self.js_windows_jdk19 + self.deploy_daily_vm_windows_jdk19 + self.deploy_graalvm_base_windows_amd64("java19") + {name: 'daily-deploy-vm-base-java19-windows-amd64', notify_groups:: ["deploy"]},
-  deploy_vm_installable_java19_windows_amd64: vm.vm_java_19 + self.svm_common_windows_amd64("19") + self.js_windows_jdk19 + self.sulong_windows + self.deploy_daily_vm_windows_jdk19 + self.deploy_graalvm_installables_windows_amd64("java19") + {name: 'daily-deploy-vm-installable-java19-windows-amd64', diskspace_required: "31GB", notify_groups:: ["deploy"]},
+  deploy_vm_base_java17_windows_amd64: vm.vm_java_17 + self.svm_common_windows_amd64("17") + self.js_windows_jdk17 + self.deploy_daily_vm_windows_jdk17 + self.deploy_graalvm_base_windows_amd64("java17") + self.deploy_build + {name: 'daily-deploy-vm-base-java17-windows-amd64', notify_groups:: ["deploy"]},
+  deploy_vm_installable_java17_windows_amd64: vm.vm_java_17 + self.svm_common_windows_amd64("17") + self.js_windows_jdk17 + self.sulong_windows + self.deploy_daily_vm_windows_jdk17 + self.deploy_graalvm_installables_windows_amd64("java17") + self.deploy_build + {name: 'daily-deploy-vm-installable-java17-windows-amd64', diskspace_required: "31GB", notify_groups:: ["deploy"]},
+  deploy_vm_base_java19_windows_amd64: vm.vm_java_19 + self.svm_common_windows_amd64("19") + self.js_windows_jdk19 + self.deploy_daily_vm_windows_jdk19 + self.deploy_graalvm_base_windows_amd64("java19") + self.deploy_build + {name: 'daily-deploy-vm-base-java19-windows-amd64', notify_groups:: ["deploy"]},
+  deploy_vm_installable_java19_windows_amd64: vm.vm_java_19 + self.svm_common_windows_amd64("19") + self.js_windows_jdk19 + self.sulong_windows + self.deploy_daily_vm_windows_jdk19 + self.deploy_graalvm_installables_windows_amd64("java19") + self.deploy_build + {name: 'daily-deploy-vm-installable-java19-windows-amd64', diskspace_required: "31GB", notify_groups:: ["deploy"]},
 
   #
   # Deploy the GraalVM Ruby image (GraalVM Base + ruby - js)
diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet
@@ -51,13 +51,15 @@ local jdks = common_json.jdks;
     mx_cmd_base:: ['mx', '--dynamicimports', '/tools,/compiler,/graal-js,/espresso,/substratevm', '--disable-installables=true', '--force-bash-launcher=true', '--skip-libraries=true'],
     build:: self.mx_cmd_base + ['build'],
     deploy:: self.mx_cmd_base + ['--suite', 'compiler', '--suite', 'truffle', '--suite', 'sdk', '--suite', 'tools', '--suite', 'regex', '--suite', 'graal-js', '--suite', 'espresso', '--suite', 'substratevm', 'maven-deploy', '--skip', $.maven_17_19_only_native.native_distributions, '--tags=default', '--all-distribution-types', '--validate', 'full', '--licenses', 'GPLv2-CPE,UPL,MIT'],
+
   },
 
   maven_17_19_only_native:: self.maven_17_19 + {
     native_distributions:: 'TRUFFLE_NFI_NATIVE,SVM_HOSTED_NATIVE',
     mx_cmd_base:: ['mx', '--dynamicimports', '/substratevm', '--disable-installables=true', '--force-bash-launcher=true', '--skip-libraries=true'],
     build:: self.mx_cmd_base + ['build', '--dependencies', self.native_distributions],
     deploy:: self.mx_cmd_base + ['maven-deploy', '--only', self.native_distributions, '--tags=default', '--all-suites', '--all-distribution-types', '--validate', 'full', '--licenses', 'GPLv2-CPE,UPL,MIT'],
+
   },
 
   notify_releaser_build: vm_common.common_vm_linux + graal_common.linux_amd64 + {
@@ -135,78 +137,48 @@ local jdks = common_json.jdks;
      name: 'gate-vm-build-without-vcs-linux-amd64',
     },
     vm_common.linux_deploy + vm_common.gate_vm_linux_amd64 + self.maven_17_19 + vm_common.sulong_linux + {
-     run: [
-       $.maven_17_19.build,
-       $.maven_17_19.deploy + ['--dry-run', 'lafo-maven'],
-     ],
+     run: [ $.maven_17_19.build, $.maven_17_19.deploy],
      name: 'gate-vm-maven-dry-run-linux-amd64',
     },
     vm_common.linux_deploy + vm_common.deploy_vm_linux_amd64 + self.maven_17_19 + vm_common.sulong_linux + {
-     run: [
-       $.maven_17_19.build,
-       $.maven_17_19.deploy + ['lafo-maven'],
-     ],
+     run: [ $.maven_17_19.build, $.maven_17_19.deploy],
      name: 'post-merge-deploy-vm-maven-linux-amd64',
      timelimit: '45:00',
      notify_groups:: ['deploy'],
     },
     vm_common.linux_deploy + vm_common.gate_vm_linux_aarch64 + self.maven_17_19_only_native + {
-     run: [
-       $.maven_17_19_only_native.build,
-       $.maven_17_19_only_native.deploy + ['--dry-run', 'lafo-maven'],
-     ],
+     run: [ $.maven_17_19_only_native.build, $.maven_17_19_only_native.deploy],
      name: 'gate-vm-maven-dry-run-linux-aarch64',
     },
     vm_common.linux_deploy + vm_common.deploy_vm_linux_aarch64 + self.maven_17_19_only_native + {
-     run: [
-       $.maven_17_19_only_native.build,
-       $.maven_17_19_only_native.deploy + ['lafo-maven'],
-     ],
+     run: [ $.maven_17_19_only_native.build, $.maven_17_19_only_native.deploy],
      name: 'post-merge-deploy-vm-maven-linux-aarch64',
      notify_groups:: ['deploy'],
     },
     vm_common.darwin_deploy + vm_common.gate_vm_darwin_amd64 + self.maven_17_19_only_native + {
-     run: [
-       $.maven_17_19_only_native.build,
-       $.maven_17_19_only_native.deploy + ['--dry-run', 'lafo-maven'],
-     ],
+     run: [ $.maven_17_19_only_native.build, $.maven_17_19_only_native.deploy],
      name: 'gate-vm-maven-dry-run-darwin-amd64',
     },
     vm_common.darwin_deploy + vm_common.gate_vm_darwin_aarch64 + self.maven_17_19_only_native + {
-     run: [
-       $.maven_17_19_only_native.build,
-       $.maven_17_19_only_native.deploy + ['--dry-run', 'lafo-maven'],
-     ],
+     run: [ $.maven_17_19_only_native.build, $.maven_17_19_only_native.deploy],
      name: 'gate-vm-maven-dry-run-darwin-aarch64',
     },
     vm_common.darwin_deploy + vm_common.deploy_daily_vm_darwin_amd64 + self.maven_17_19_only_native + {
-     run: [
-       $.maven_17_19_only_native.build,
-       $.maven_17_19_only_native.deploy + ['lafo-maven'],
-     ],
+     run: [ $.maven_17_19_only_native.build, $.maven_17_19_only_native.deploy],
      name: 'daily-deploy-vm-maven-darwin-amd64',
      notify_groups:: ['deploy'],
     },
     vm_common.darwin_deploy + vm_common.deploy_daily_vm_darwin_aarch64 + self.maven_17_19_only_native + {
-     run: [
-       $.maven_17_19_only_native.build,
-       $.maven_17_19_only_native.deploy + ['lafo-maven'],
-     ],
+     run: [ $.maven_17_19_only_native.build, $.maven_17_19_only_native.deploy],
      name: 'daily-deploy-vm-maven-darwin-aarch64',
      notify_groups:: ['deploy'],
     },
     vm_common.svm_common_windows_amd64("17") + vm_common.gate_vm_windows_amd64 + self.maven_17_19_only_native + {
-     run: [
-       $.maven_17_19_only_native.build,
-       $.maven_17_19_only_native.deploy + ['--dry-run', 'lafo-maven'],
-     ],
+     run: [ $.maven_17_19_only_native.build, $.maven_17_19_only_native.deploy],
      name: 'gate-vm-maven-dry-run-windows-amd64',
     },
     vm_common.svm_common_windows_amd64("17") + vm_common.deploy_daily_vm_windows + self.maven_17_19_only_native + {
-     run: [
-       $.maven_17_19_only_native.build,
-       $.maven_17_19_only_native.deploy + ['lafo-maven'],
-     ],
+     run: [ $.maven_17_19_only_native.build, $.maven_17_19_only_native.deploy],
      name: 'daily-deploy-vm-maven-windows-amd64',
      notify_groups:: ['deploy'],
     },

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"README": "This file contains definitions that are useful for the hocon and jsonnet CI files of multiple repositories.",`
`3`	`3`
`4`		`- "mx_version" : "6.11.4",`
	`4`	`+ "mx_version" : "6.12.7",`
`5`	`5`
`6`	`6`	`"jdks": {`
`7`	`7`	`"openjdk11": {"name": "jpg-jdk", "version": "11.0.11", "build_id": "9", "open": true, "release": true, "platformspecific": true },`