Recompress xz -> xz with tuned compression options

This adds support for recompressing the .xz tarballs in promote-release,
with a goal of moving rust-lang/rust CI back to producing "balanced"
profile binaries rather than the highly compressed ones currently
produced. This will cut rust-lang/rust CI times while keeping the
production artifacts in static.r-l.o equivalently compressed.

This PR doesn't enable the .xz recompression by default, that will be
done in a follow up simpleinfra push. Depending on the impact to
(especially) stable build times we may wish to bump the promote-release
production container from 8 cores to 72 cores (the next largest
container) so that we still finish in a reasonable amount of time.

Author: Mark-Simulacrum

local/run.sh

@@ -118,6 +118,8 @@ export PROMOTE_RELEASE_UPLOAD_ADDR="http://localhost:9000/static"
 export PROMOTE_RELEASE_UPLOAD_BUCKET="static"
 export PROMOTE_RELEASE_UPLOAD_STORAGE_CLASS="STANDARD"
 export PROMOTE_RELEASE_UPLOAD_DIR="dist"
+# Enable xz recompression to check it in CI
+export PROMOTE_RELEASE_XZ_RECOMPRESS=1
 # Environment variables used only by local releases
 export PROMOTE_RELEASE_BYPASS_STARTUP_CHECKS="1"
 export PROMOTE_RELEASE_GZIP_COMPRESSION_LEVEL="1" # Faster recompressions

src/config.rs

@@ -125,8 +125,12 @@ pub(crate) struct Config {
     /// * Preventing multiple releases on stable and beta of the same version number.
     pub(crate) bypass_startup_checks: bool,
 
-    /// Whether to force the recompression of .gz files into .xz.
-    pub(crate) wip_recompress: bool,
+    /// Whether to force the recompression from input tarballs into .gz compressed tarballs.
+    ///
+    /// This is on by default if .gz tarballs aren't available in the input.
+    pub(crate) recompress_gz: bool,
+    /// Whether to force the recompression from input tarballs into highly compressed .xz tarballs.
+    pub(crate) recompress_xz: bool,
 
     /// The compression level to use when recompressing tarballs with gzip.
     pub(crate) gzip_compression_level: u32,
@@ -212,7 +216,8 @@ impl Config {
             upload_bucket: require_env("UPLOAD_BUCKET")?,
             storage_class: default_env("UPLOAD_STORAGE_CLASS", "INTELLIGENT_TIERING".into())?,
             upload_dir: require_env("UPLOAD_DIR")?,
-            wip_recompress: bool_env("WIP_RECOMPRESS")?,
+            recompress_xz: bool_env("RECOMPRESS_XZ")?,
+            recompress_gz: bool_env("RECOMPRESS_GZ")?,
             rustc_tag_repository: maybe_env("RUSTC_TAG_REPOSITORY")?,
             blog_repository: maybe_env("BLOG_REPOSITORY")?,
             blog_pr: maybe_env("BLOG_MERGE_PR")?,

src/main.rs

@@ -380,14 +380,11 @@ impl Context {
                 Some("asc") | Some("sha256") => {
                     fs::remove_file(&path)?;
                 }
-                // Generate *.gz from *.xz...
+                // Store off the input files for potential recompression.
                 Some("xz") => {
-                    let gz_path = path.with_extension("gz");
-                    if self.config.wip_recompress || !gz_path.is_file() {
-                        to_recompress.push((path.to_path_buf(), gz_path));
-                    }
+                    to_recompress.push(path.to_path_buf());
                 }
-                Some("gz") if self.config.wip_recompress => {
+                Some("gz") if self.config.recompress_gz => {
                     fs::remove_file(&path)?;
                 }
                 _ => {}
@@ -404,19 +401,76 @@ impl Context {
                 to_recompress.len(),
                 to_recompress.len().min(rayon::current_num_threads()),
             );
+            println!("gzip recompression enabled: {}", self.config.recompress_gz);
+            println!("xz recompression enabled: {}", self.config.recompress_xz);
             let recompress_start = Instant::now();
 
+            let recompress_gz = self.config.recompress_gz;
+            let recompress_xz = self.config.recompress_xz;
             let compression_level = flate2::Compression::new(self.config.gzip_compression_level);
             to_recompress
                 .par_iter()
-                .map(|(xz_path, gz_path)| {
-                    println!("recompressing {}...", gz_path.display());
-
-                    let xz = File::open(xz_path)?;
-                    let mut xz = XzDecoder::new(xz);
-                    let gz = File::create(gz_path)?;
-                    let mut gz = flate2::write::GzEncoder::new(gz, compression_level);
-                    io::copy(&mut xz, &mut gz)?;
+                .map(|xz_path| {
+                    println!("recompressing {}...", xz_path.display());
+                    let gz_path = xz_path.with_extension("gz");
+
+                    // Produce gzip if explicitly enabled or the destination file doesn't exist.
+                    if recompress_gz || !gz_path.is_file() {
+                        let mut xz_orig = XzDecoder::new(File::open(xz_path)?);
+                        let gz = File::create(gz_path)?;
+                        let mut gz = flate2::write::GzEncoder::new(gz, compression_level);
+                        io::copy(&mut xz_orig, &mut gz)?;
+                    }
+
+                    // xz recompression with more aggressive settings than we want to take the time
+                    // for in rust-lang/rust CI. This cuts 5-15% off of the produced tarballs.
+                    //
+                    // Note that this is using a single-threaded compressor as we're parallelizing
+                    // via rayon already. In rust-lang/rust we were trying to use parallel
+                    // compression, but the default block size for that is 3*dict_size so we
+                    // weren't actually using more than one core in most of the builders with
+                    // <192MB uncompressed tarballs. In promote-release since we're recompressing
+                    // 100s of tarballs there's no need for each individual compression to be
+                    // parallel.
+                    if recompress_xz {
+                        let mut filters = xz2::stream::Filters::new();
+                        let mut lzma_ops = xz2::stream::LzmaOptions::new_preset(9).unwrap();
+                        // This sets the overall dictionary size, which is also how much memory (baseline)
+                        // is needed for decompression.
+                        lzma_ops.dict_size(64 * 1024 * 1024);
+                        // Use the best match finder for compression ratio.
+                        lzma_ops.match_finder(xz2::stream::MatchFinder::BinaryTree4);
+                        lzma_ops.mode(xz2::stream::Mode::Normal);
+                        // Set nice len to the maximum for best compression ratio
+                        lzma_ops.nice_len(273);
+                        // Set depth to a reasonable value, 0 means auto, 1000 is somwhat high but gives
+                        // good results.
+                        lzma_ops.depth(1000);
+                        // 2 is the default and does well for most files
+                        lzma_ops.position_bits(2);
+                        // 0 is the default and does well for most files
+                        lzma_ops.literal_position_bits(0);
+                        // 3 is the default and does well for most files
+                        lzma_ops.literal_context_bits(3);
+
+                        filters.lzma2(&lzma_ops);
+
+                        // FIXME: Do we want a checksum as part of compression?
+                        let stream = xz2::stream::Stream::new_stream_encoder(
+                            &filters,
+                            xz2::stream::Check::None,
+                        )
+                        .unwrap();
+                        let xz_recompressed = xz_path.with_extension("xz_recompressed");
+                        let xz_out = File::create(&xz_recompressed)?;
+                        let mut xz_out = xz2::write::XzEncoder::new_stream(
+                            std::io::BufWriter::new(xz_out),
+                            stream,
+                        );
+                        let mut xz_orig = XzDecoder::new(File::open(xz_path)?);
+                        io::copy(&mut xz_orig, &mut xz_out)?;
+                        fs::rename(&xz_recompressed, &xz_path)?;
+                    }
 
                     Ok::<(), Error>(())
                 })