Unrolled build for rust-lang#127463

Rollup merge of rust-lang#127463 - onur-ozkan:precompiled-rustdoc, r=Kobzol

 use precompiled rustdoc with CI rustc

When CI rustc is enabled and rustdoc sources are unchanged, we can use the precompiled rustdoc from the CI rustc's sysroot. This speeds up bootstrapping quite a lot by avoiding unnecessary rustdoc compilation.

Author: rust-timer

src/bootstrap/src/core/build_steps/llvm.rs

@@ -26,7 +26,6 @@ use crate::{generate_smart_stamp_hash, CLang, GitRepo, Kind};
 
 use crate::utils::exec::command;
 use build_helper::ci::CiEnv;
-use build_helper::git::get_git_merge_base;
 
 #[derive(Clone)]
 pub struct LlvmResult {
@@ -154,26 +153,18 @@ pub fn prebuilt_llvm_config(builder: &Builder<'_>, target: TargetSelection) -> L
 /// This retrieves the LLVM sha we *want* to use, according to git history.
 pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String {
     let llvm_sha = if is_git {
-        // We proceed in 2 steps. First we get the closest commit that is actually upstream. Then we
-        // walk back further to the last bors merge commit that actually changed LLVM. The first
-        // step will fail on CI because only the `auto` branch exists; we just fall back to `HEAD`
-        // in that case.
-        let closest_upstream = get_git_merge_base(&config.git_config(), Some(&config.src))
-            .unwrap_or_else(|_| "HEAD".into());
-        let mut rev_list = helpers::git(Some(&config.src));
-        rev_list.args(&[
-            PathBuf::from("rev-list"),
-            format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(),
-            "-n1".into(),
-            "--first-parent".into(),
-            closest_upstream.into(),
-            "--".into(),
-            config.src.join("src/llvm-project"),
-            config.src.join("src/bootstrap/download-ci-llvm-stamp"),
-            // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
-            config.src.join("src/version"),
-        ]);
-        output(rev_list.as_command_mut()).trim().to_owned()
+        helpers::get_closest_merge_base_commit(
+            Some(&config.src),
+            &config.git_config(),
+            &config.stage0_metadata.config.git_merge_commit_email,
+            &[
+                config.src.join("src/llvm-project"),
+                config.src.join("src/bootstrap/download-ci-llvm-stamp"),
+                // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
+                config.src.join("src/version"),
+            ],
+        )
+        .unwrap()
     } else if let Some(info) = channel::read_commit_info_file(&config.src) {
         info.sha.trim().to_owned()
     } else {

src/bootstrap/src/core/build_steps/tool.rs

@@ -9,7 +9,7 @@ use crate::core::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun,
 use crate::core::config::TargetSelection;
 use crate::utils::channel::GitInfo;
 use crate::utils::exec::{command, BootstrapCommand};
-use crate::utils::helpers::{add_dylib_path, exe, t};
+use crate::utils::helpers::{add_dylib_path, exe, get_closest_merge_base_commit, git, t};
 use crate::Compiler;
 use crate::Mode;
 use crate::{gha, Kind};
@@ -554,6 +554,57 @@ impl Step for Rustdoc {
         }
         let target = target_compiler.host;
 
+        let bin_rustdoc = || {
+            let sysroot = builder.sysroot(target_compiler);
+            let bindir = sysroot.join("bin");
+            t!(fs::create_dir_all(&bindir));
+            let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host));
+            let _ = fs::remove_file(&bin_rustdoc);
+            bin_rustdoc
+        };
+
+        // If CI rustc is enabled and we haven't modified the rustdoc sources,
+        // use the precompiled rustdoc from CI rustc's sysroot to speed up bootstrapping.
+        if builder.download_rustc()
+            && target_compiler.stage > 0
+            && builder.rust_info().is_managed_git_subrepository()
+        {
+            let commit = get_closest_merge_base_commit(
+                Some(&builder.config.src),
+                &builder.config.git_config(),
+                &builder.config.stage0_metadata.config.git_merge_commit_email,
+                &[],
+            )
+            .unwrap();
+
+            let librustdoc_src = builder.config.src.join("src/librustdoc");
+            let rustdoc_src = builder.config.src.join("src/tools/rustdoc");
+
+            // FIXME: The change detection logic here is quite similar to `Config::download_ci_rustc_commit`.
+            // It would be better to unify them.
+            let has_changes = !git(Some(&builder.config.src))
+                .allow_failure()
+                .run_always()
+                .args(["diff-index", "--quiet", &commit])
+                .arg("--")
+                .arg(librustdoc_src)
+                .arg(rustdoc_src)
+                .run(builder)
+                .is_success();
+
+            if !has_changes {
+                let precompiled_rustdoc = builder
+                    .config
+                    .ci_rustc_dir()
+                    .join("bin")
+                    .join(exe("rustdoc", target_compiler.host));
+
+                let bin_rustdoc = bin_rustdoc();
+                builder.copy_link(&precompiled_rustdoc, &bin_rustdoc);
+                return bin_rustdoc;
+            }
+        }
+
         let build_compiler = if builder.download_rustc() && target_compiler.stage == 1 {
             // We already have the stage 1 compiler, we don't need to cut the stage.
             builder.compiler(target_compiler.stage, builder.config.build)
@@ -614,11 +665,7 @@ impl Step for Rustdoc {
 
         // don't create a stage0-sysroot/bin directory.
         if target_compiler.stage > 0 {
-            let sysroot = builder.sysroot(target_compiler);
-            let bindir = sysroot.join("bin");
-            t!(fs::create_dir_all(&bindir));
-            let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host));
-            let _ = fs::remove_file(&bin_rustdoc);
+            let bin_rustdoc = bin_rustdoc();
             builder.copy_link(&tool_rustdoc, &bin_rustdoc);
             bin_rustdoc
         } else {

src/bootstrap/src/core/config/config.rs

@@ -20,7 +20,7 @@ use crate::core::build_steps::llvm;
 use crate::core::config::flags::{Color, Flags, Warnings};
 use crate::utils::cache::{Interned, INTERNER};
 use crate::utils::channel::{self, GitInfo};
-use crate::utils::helpers::{self, exe, output, t};
+use crate::utils::helpers::{self, exe, get_closest_merge_base_commit, output, t};
 use build_helper::exit;
 use serde::{Deserialize, Deserializer};
 use serde_derive::Deserialize;
@@ -2471,14 +2471,13 @@ impl Config {
 
         // Look for a version to compare to based on the current commit.
         // Only commits merged by bors will have CI artifacts.
-        let merge_base = output(
-            helpers::git(Some(&self.src))
-                .arg("rev-list")
-                .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email))
-                .args(["-n1", "--first-parent", "HEAD"])
-                .as_command_mut(),
-        );
-        let commit = merge_base.trim_end();
+        let commit = get_closest_merge_base_commit(
+            Some(&self.src),
+            &self.git_config(),
+            &self.stage0_metadata.config.git_merge_commit_email,
+            &[],
+        )
+        .unwrap();
         if commit.is_empty() {
             println!("ERROR: could not find commit hash for downloading rustc");
             println!("HELP: maybe your repository history is too shallow?");
@@ -2489,7 +2488,7 @@ impl Config {
 
         // Warn if there were changes to the compiler or standard library since the ancestor commit.
         let has_changes = !t!(helpers::git(Some(&self.src))
-            .args(["diff-index", "--quiet", commit])
+            .args(["diff-index", "--quiet", &commit])
             .arg("--")
             .args([self.src.join("compiler"), self.src.join("library")])
             .as_command_mut()
@@ -2565,14 +2564,13 @@ impl Config {
     ) -> Option<String> {
         // Look for a version to compare to based on the current commit.
         // Only commits merged by bors will have CI artifacts.
-        let merge_base = output(
-            helpers::git(Some(&self.src))
-                .arg("rev-list")
-                .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email))
-                .args(["-n1", "--first-parent", "HEAD"])
-                .as_command_mut(),
-        );
-        let commit = merge_base.trim_end();
+        let commit = get_closest_merge_base_commit(
+            Some(&self.src),
+            &self.git_config(),
+            &self.stage0_metadata.config.git_merge_commit_email,
+            &[],
+        )
+        .unwrap();
         if commit.is_empty() {
             println!("error: could not find commit hash for downloading components from CI");
             println!("help: maybe your repository history is too shallow?");
@@ -2583,7 +2581,7 @@ impl Config {
 
         // Warn if there were changes to the compiler or standard library since the ancestor commit.
         let mut git = helpers::git(Some(&self.src));
-        git.args(["diff-index", "--quiet", commit, "--"]);
+        git.args(["diff-index", "--quiet", &commit, "--"]);
 
         // Handle running from a directory other than the top level
         let top_level = &self.src;

src/bootstrap/src/utils/helpers.rs

@@ -3,6 +3,7 @@
 //! Simple things like testing the various filesystem operations here and there,
 //! not a lot of interesting happenings here unfortunately.
 
+use build_helper::git::{get_git_merge_base, output_result, GitConfig};
 use build_helper::util::fail;
 use std::env;
 use std::ffi::OsStr;
@@ -521,3 +522,26 @@ pub fn git(source_dir: Option<&Path>) -> BootstrapCommand {
 
     git
 }
+
+/// Returns the closest commit available from upstream for the given `author` and `target_paths`.
+///
+/// If it fails to find the commit from upstream using `git merge-base`, fallbacks to HEAD.
+pub fn get_closest_merge_base_commit(
+    source_dir: Option<&Path>,
+    config: &GitConfig<'_>,
+    author: &str,
+    target_paths: &[PathBuf],
+) -> Result<String, String> {
+    let mut git = git(source_dir).capture_stdout();
+
+    let merge_base = get_git_merge_base(config, source_dir).unwrap_or_else(|_| "HEAD".into());
+
+    git.arg(Path::new("rev-list"));
+    git.args([&format!("--author={author}"), "-n1", "--first-parent", &merge_base]);
+
+    if !target_paths.is_empty() {
+        git.arg("--").args(target_paths);
+    }
+
+    Ok(output_result(git.as_command_mut())?.trim().to_owned())
+}

src/tools/build_helper/src/git.rs

@@ -7,7 +7,7 @@ pub struct GitConfig<'a> {
 }
 
 /// Runs a command and returns the output
-fn output_result(cmd: &mut Command) -> Result<String, String> {
+pub fn output_result(cmd: &mut Command) -> Result<String, String> {
     let output = match cmd.stderr(Stdio::inherit()).output() {
         Ok(status) => status,
         Err(e) => return Err(format!("failed to run command: {:?}: {}", cmd, e)),