From 7b3a0828aefe46173236a01cdba188043124ed4c Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Tue, 30 Jul 2019 09:15:26 -0400 Subject: [PATCH 1/2] Clear target directory at 80% disk usage This avoids filling up the disk and leading to postgres failure. --- Cargo.lock | 31 ++++++++++++++++++++++++++ Cargo.toml | 1 + src/docbuilder/chroot_builder.rs | 38 ++++++++++++++++++++++++++++++-- src/lib.rs | 1 + 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57a00288b..402b4a34c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -208,6 +208,11 @@ dependencies = [ "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bytesize" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "bytesize" version = "1.0.0" @@ -450,6 +455,7 @@ dependencies = [ "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "slug 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "staticfile 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "systemstat 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1422,6 +1428,14 @@ name = "nodrop" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "nom" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num" version = "0.1.42" @@ -2549,6 +2563,20 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "systemstat" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytesize 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tar" version = "0.4.20" @@ -3152,6 +3180,7 @@ dependencies = [ "checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" "checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +"checksum bytesize 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "16d794c5fe594cfa8fbe8ae274de4048176c69f2d9ac5e637166e73b71d460b8" "checksum bytesize 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "716960a18f978640f25101b5cbf1c6f6b0d3192fab36a2d98ca96f0ecbe41010" "checksum cargo 0.39.0 (git+https://github.com/rust-lang/cargo.git)" = "" "checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" @@ -3275,6 +3304,7 @@ dependencies = [ "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" +"checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" "checksum num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" "checksum num-bigint 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)" = "e63899ad0da84ce718c14936262a41cee2c79c981fc0a0e7c7beb47d5a07e8c1" "checksum num-complex 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "b288631d7878aaf59442cffd36910ea604ecd7745c36054328595114001c9656" @@ -3401,6 +3431,7 @@ dependencies = [ "checksum syn 0.15.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b036b7b35e846707c0e55c2c9441fa47867c0f87fca416921db3261b1d8c741a" "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum synstructure 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ec37f4fab4bafaf6b5621c1d54e6aa5d4d059a8f84929e87abfdd7f9f04c6db2" +"checksum systemstat 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "11374381f619810a32d086459e740a0e4a683f15beea3fe5f3cddb40c8791106" "checksum tar 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)" = "a303ba60a099fcd2aaa646b14d2724591a96a75283e4b7ed3d1a1658909d9ae2" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" "checksum tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "55c1195ef8513f3273d55ff59fe5da6940287a0d7a98331254397f464833675b" diff --git a/Cargo.toml b/Cargo.toml index b8f2df296..6f22ad43c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ rusoto_core = "0.40" rusoto_credential = "0.40" futures = "0.1" tokio = "0.1" +systemstat = "0.1.4" # iron dependencies iron = "0.5" diff --git a/src/docbuilder/chroot_builder.rs b/src/docbuilder/chroot_builder.rs index c9a90665e..aea0fe6c5 100644 --- a/src/docbuilder/chroot_builder.rs +++ b/src/docbuilder/chroot_builder.rs @@ -8,12 +8,14 @@ use db::{connect_db, add_package_into_database, add_build_into_database, add_pat use cargo::core::Package; use cargo::util::CargoResultExt; use std::process::Command; -use std::path::PathBuf; -use std::fs::remove_dir_all; +use std::path::{Path, PathBuf}; +use std::fs::{self, remove_dir_all}; use postgres::Connection; use rustc_serialize::json::{Json, ToJson}; use error::Result; +use systemstat::{Platform, Filesystem, System}; +const MAX_DISK_USAGE: f32 = 80.0; /// List of targets supported by docs.rs const TARGETS: [&'static str; 6] = [ @@ -237,6 +239,20 @@ impl DocBuilder { .join("doc"); let _ = remove_dir_all(crate_doc_path); } + + let crate_build_path = PathBuf::from(&self.options.chroot_path) + .join("home") + .join(&self.options.chroot_user) + .join("cratesfyi"); + let fs = mount_for_path(&crate_build_path); + let disk_usage = 100.0 - 100.0 * (fs.free.as_usize() as f32 / fs.total.as_usize() as f32); + if disk_usage >= MAX_DISK_USAGE { + info!("Cleaning target directory, disk usage {:.2} exceeded {:.2}", + disk_usage, MAX_DISK_USAGE); + let _ = remove_dir_all(&crate_build_path); + let _ = fs::create_dir_all(&crate_build_path); + } + Ok(()) } @@ -466,6 +482,24 @@ fn crates(path: PathBuf, mut func: F) -> Result<()> crates_from_path(&path, &mut func) } +fn mount_for_path(path: &Path) -> Filesystem { + let system = System::new(); + + let mut found = None; + let mut found_pos = std::usize::MAX; + for mount in system.mounts().expect("get all mounts").into_iter() { + let mount_path = Path::new(&mount.fs_mounted_on); + for (i, ancestor) in path.ancestors().enumerate() { + if ancestor == mount_path && i < found_pos { + found_pos = i; + found = Some(mount); + break; + } + } + } + found.expect("on a disk mount") +} + #[cfg(test)] mod test { diff --git a/src/lib.rs b/src/lib.rs index e9f964940..f7d28990e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,7 @@ extern crate rusoto_core; extern crate rusoto_credential; extern crate futures; extern crate tokio; +extern crate systemstat; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::ChrootBuilderResult; From 442b5c5f46bcea012680384e46b18192f44d6101 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Tue, 30 Jul 2019 09:16:16 -0400 Subject: [PATCH 2/2] Remove unused variable --- src/db/file.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/file.rs b/src/db/file.rs index 2bb9092f1..26689c36c 100644 --- a/src/db/file.rs +++ b/src/db/file.rs @@ -72,7 +72,7 @@ pub fn get_path(conn: &Connection, path: &str) -> Option { let res = match res { Ok(r) => r, - Err(err) => { + Err(_) => { return None; } };