Auto merge of #127050 - Kobzol:reproducibility-git, r=onur-ozkan

Make mtime of reproducible tarballs dependent on git commit

Since #123246, our tarballs should be fully reproducible. That means that the mtime of all files and directories in the tarballs is set to the date of the first Rust commit (from 2006). However, this is causing some mtime invalidation issues (#125578 (comment)).

Ideally, we would like to keep the mtime reproducible, but still update it with new versions of Rust. That's what this PR does. It modifies the tarball installer bootstrap invocation so that if the current rustc directory is managed by git, we will set the UTC timestamp of the latest commit as the mtime for all files in the archive. This means that the archive should be still fully reproducible from a given commit SHA, but it will also be changed with new beta bumps and `download-rustc` versions.

Note that only files are set to this mtime, directories are still set to the year 2006, because the `tar` library used by `rust-installer` doesn't allow us to selectively override mtime for directories (or at least I haven't found it). We could work around that by doing all the mtime modifications in bootstrap, but that would require more changes. I think/hope that just modifying the file mtimes should be enough. It should at least fix cargo `rustc` mtime invalidation.

Fixes: #125578

r? `@onur-ozkan`

try-job: x86_64-gnu-distcheck

Author: bors

src/bootstrap/src/utils/tarball.rs

@@ -9,9 +9,9 @@ use std::path::{Path, PathBuf};
 
 use crate::core::builder::Builder;
 use crate::core::{build_steps::dist::distdir, builder::Kind};
-use crate::utils::channel;
 use crate::utils::exec::BootstrapCommand;
 use crate::utils::helpers::{move_file, t};
+use crate::utils::{channel, helpers};
 
 #[derive(Copy, Clone)]
 pub(crate) enum OverlayKind {
@@ -351,6 +351,30 @@ impl<'a> Tarball<'a> {
         };
 
         cmd.args(["--compression-profile", compression_profile]);
+
+        // We want to use a pinned modification time for files in the archive
+        // to achieve better reproducibility. However, using the same mtime for all
+        // releases is not ideal, because it can break e.g. Cargo mtime checking
+        // (https://github.com/rust-lang/rust/issues/125578).
+        // Therefore, we set mtime to the date of the latest commit (if we're managed
+        // by git). In this way, the archive will still be always the same for a given commit
+        // (achieving reproducibility), but it will also change between different commits and
+        // Rust versions, so that it won't break mtime-based caches.
+        //
+        // Note that this only overrides the mtime of files, not directories, due to the
+        // limitations of the tarballer tool. Directories will have their mtime set to 2006.
+
+        // Get the UTC timestamp of the last git commit, if we're under git.
+        // We need to use UTC, so that anyone who tries to rebuild from the same commit
+        // gets the same timestamp.
+        if self.builder.rust_info().is_managed_git_subrepository() {
+            // %ct means committer date
+            let timestamp = helpers::output(
+                helpers::git(Some(&self.builder.src)).arg("log").arg("-1").arg("--format=%ct"),
+            );
+            cmd.args(["--override-file-mtime", timestamp.trim()]);
+        }
+
         self.builder.run(cmd);
 
         // Ensure there are no symbolic links in the tarball. In particular,

src/tools/rust-installer/src/combiner.rs

@@ -55,6 +55,12 @@ actor! {
         /// The formats used to compress the tarball
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -145,7 +151,8 @@ impl Combiner {
             .input(self.package_name)
             .output(path_to_str(&output)?.into())
             .compression_profile(self.compression_profile)
-            .compression_formats(self.compression_formats);
+            .compression_formats(self.compression_formats)
+            .override_file_mtime(self.override_file_mtime);
         tarballer.run()?;
 
         Ok(())

src/tools/rust-installer/src/generator.rs

@@ -61,6 +61,12 @@ actor! {
         /// The formats used to compress the tarball
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -114,7 +120,8 @@ impl Generator {
             .input(self.package_name)
             .output(path_to_str(&output)?.into())
             .compression_profile(self.compression_profile)
-            .compression_formats(self.compression_formats);
+            .compression_formats(self.compression_formats)
+            .override_file_mtime(self.override_file_mtime);
         tarballer.run()?;
 
         Ok(())

src/tools/rust-installer/src/tarballer.rs

@@ -32,6 +32,12 @@ actor! {
         /// The formats used to compress the tarball.
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -65,6 +71,8 @@ impl Tarballer {
         let buf = BufWriter::with_capacity(1024 * 1024, encoder);
         let mut builder = Builder::new(buf);
         // Make uid, gid and mtime deterministic to improve reproducibility
+        // The modification time of directories will be set to the date of the first Rust commit.
+        // The modification time of files will be set to `override_file_mtime` (see `append_path`).
         builder.mode(HeaderMode::Deterministic);
 
         let pool = rayon::ThreadPoolBuilder::new().num_threads(2).build().unwrap();
@@ -77,7 +85,7 @@ impl Tarballer {
             }
             for path in files {
                 let src = Path::new(&self.work_dir).join(&path);
-                append_path(&mut builder, &src, &path)
+                append_path(&mut builder, &src, &path, self.override_file_mtime)
                     .with_context(|| format!("failed to tar file '{}'", src.display()))?;
             }
             builder
@@ -93,10 +101,16 @@ impl Tarballer {
     }
 }
 
-fn append_path<W: Write>(builder: &mut Builder<W>, src: &Path, path: &String) -> Result<()> {
+fn append_path<W: Write>(
+    builder: &mut Builder<W>,
+    src: &Path,
+    path: &String,
+    override_file_mtime: u64,
+) -> Result<()> {
     let stat = symlink_metadata(src)?;
     let mut header = Header::new_gnu();
     header.set_metadata_in_mode(&stat, HeaderMode::Deterministic);
+    header.set_mtime(override_file_mtime);
 
     if stat.file_type().is_symlink() {
         let link = read_link(src)?;