From 9776e7005d9c8e9f0b9fe138f16f8a5d8957dcab Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Thu, 22 Aug 2019 02:28:06 +0300 Subject: [PATCH 01/23] WIP rustwide --- Cargo.lock | 62 ++++++++++++++++++++ Cargo.toml | 1 + src/bin/cratesfyi.rs | 23 +++++++- src/lib.rs | 1 + src/utils/build_doc.rs | 2 +- src/utils/build_doc_rustwide.rs | 101 ++++++++++++++++++++++++++++++++ src/utils/mod.rs | 4 +- 7 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 src/utils/build_doc_rustwide.rs diff --git a/Cargo.lock b/Cargo.lock index 89e56cff8..9e257463f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -451,6 +451,7 @@ dependencies = [ "rusoto_credential 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", "rusoto_s3 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "rustwide 0.1.0 (git+https://github.com/onur/crater.git?rev=6907da8)", "sass-rs 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama_postgres 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1425,6 +1426,18 @@ dependencies = [ "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "nix" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nodrop" version = "0.1.12" @@ -1615,6 +1628,11 @@ name = "percent-encoding" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "persistent" version = "0.3.0" @@ -2173,6 +2191,31 @@ dependencies = [ "stb_truetype 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rustwide" +version = "0.1.0" +source = "git+https://github.com/onur/crater.git?rev=6907da8#6907da82bfcbcd74a25e24c50a834aae8a552d42" +dependencies = [ + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "reqwest 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-process 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ryu" version = "0.2.6" @@ -2259,6 +2302,11 @@ name = "scopeguard" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "scopeguard" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "security-framework" version = "0.2.1" @@ -2604,6 +2652,7 @@ dependencies = [ "filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3163,6 +3212,14 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "xml-rs" version = "0.7.0" @@ -3322,6 +3379,7 @@ dependencies = [ "checksum native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8b0a7bd714e83db15676d31caf968ad7318e9cc35f93c85a90231c8f22867549" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4" +"checksum nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d37e713a259ff641624b6cb20e3b12b2952313ba36b6823c0f16e6cfd9e5de17" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" "checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" "checksum num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" @@ -3343,6 +3401,7 @@ dependencies = [ "checksum parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f0802bff09003b291ba756dc7e79313e51cc31667e94afbe847def490424cde5" "checksum parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" "checksum persistent 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c9c94f2ef72dc272c6bcc8157ccf2bc7da14f4c58c69059ac2fc48492d6916" "checksum pest 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0a6dda33d67c26f0aac90d324ab2eb7239c819fc7b2552fe9faa4fe88441edc8" "checksum pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0fce5d8b5cc33983fc74f78ad552b5522ab41442c4ca91606e4236eb4b5ceefc" @@ -3401,6 +3460,7 @@ dependencies = [ "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustfix 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "af7c21531a91512a4a51b490be6ba1c8eff34fdda0dc5bf87dc28d86748aac56" "checksum rusttype 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b8eb11f5b0a98c8eca2fb1483f42646d8c340e83e46ab416f8a063a0fd0eeb20" +"checksum rustwide 0.1.0 (git+https://github.com/onur/crater.git?rev=6907da8)" = "" "checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" @@ -3413,6 +3473,7 @@ dependencies = [ "checksum schemamama_postgres 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9a69defe7b625fa5c4bfda0a1525c9729baef68f620e505464b7bf0a4d1697f6" "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" +"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" "checksum security-framework 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "697d3f3c23a618272ead9e1fb259c1411102b31c6af8b93f1d64cca9c3b0e8e0" "checksum security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab01dfbe5756785b5b4d46e0289e5a18071dfa9a7c2b24213ea00b9ef9b665bf" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" @@ -3520,4 +3581,5 @@ dependencies = [ "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" "checksum xml-rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c1cb601d29fe2c2ac60a2b2e5e293994d87a1f6fa9687a31a15270f909be9c2" diff --git a/Cargo.toml b/Cargo.toml index fffbc96b4..28aa08da1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ tokio = "0.1" systemstat = "0.1.4" prometheus = { version = "0.7.0", default-features = false } lazy_static = "1.0.0" +rustwide = { git = "https://github.com/onur/crater.git", rev = "6907da8" } # iron dependencies iron = "0.5" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 71e6193b9..e8f3943db 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -12,7 +12,7 @@ use std::path::PathBuf; use clap::{Arg, App, SubCommand}; use cratesfyi::{DocBuilder, DocBuilderOptions, db}; -use cratesfyi::utils::{build_doc, add_crate_to_queue}; +use cratesfyi::utils::{build_doc, build_doc_rustwide, add_crate_to_queue}; use cratesfyi::start_web_server; use cratesfyi::db::{add_path_into_database, connect_db}; @@ -37,6 +37,20 @@ pub fn main() { .index(3) .required(false) .help("The target platform to compile for"))) + .subcommand(SubCommand::with_name("doc_rustwide") + .about("Builds documentation of a crate with rustwide") + .arg(Arg::with_name("CRATE_NAME") + .index(1) + .required(true) + .help("Crate name")) + .arg(Arg::with_name("CRATE_VERSION") + .index(2) + .required(true) + .help("Crate version")) + .arg(Arg::with_name("TARGET") + .index(3) + .required(false) + .help("The target platform to compile for"))) .subcommand(SubCommand::with_name("build") .about("Builds documentation in a chroot environment") .arg(Arg::with_name("PREFIX") @@ -160,6 +174,13 @@ pub fn main() { if let Err(e) = build_doc(name, version, target) { panic!("{:#?}", e); } + } else if let Some(matches) = matches.subcommand_matches("doc_rustwide") { + let name = matches.value_of("CRATE_NAME").unwrap(); + let version = matches.value_of("CRATE_VERSION").unwrap(); + let target = matches.value_of("TARGET"); + if let Err(e) = build_doc_rustwide(name, version, target) { + panic!("{:#?}", e); + } } else if let Some(matches) = matches.subcommand_matches("build") { let docbuilder_opts = { let mut docbuilder_opts = if let Some(prefix) = matches.value_of("PREFIX") { diff --git a/src/lib.rs b/src/lib.rs index 0da58ca8c..c98890cd9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,6 +40,7 @@ extern crate rusoto_credential; extern crate futures; extern crate tokio; extern crate systemstat; +extern crate rustwide; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::ChrootBuilderResult; diff --git a/src/utils/build_doc.rs b/src/utils/build_doc.rs index 7fb9d69b5..be1f41d9b 100644 --- a/src/utils/build_doc.rs +++ b/src/utils/build_doc.rs @@ -127,7 +127,7 @@ pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> Result Ok(try!(ws.current()).clone()) } -fn resolve_deps<'cfg>(pkg: &Package, config: &'cfg Config, src: Box) +pub fn resolve_deps<'cfg>(pkg: &Package, config: &'cfg Config, src: Box) -> CargoResult> { let mut registry = try!(PackageRegistry::new(config)); diff --git a/src/utils/build_doc_rustwide.rs b/src/utils/build_doc_rustwide.rs new file mode 100644 index 000000000..87c1cb1bc --- /dev/null +++ b/src/utils/build_doc_rustwide.rs @@ -0,0 +1,101 @@ +use cargo::core::{enable_nightly_features, Package, SourceId, Workspace}; +use cargo::sources::SourceConfigMap; +use cargo::util::{internal, Config}; +use error::Result; +use rustwide::{cmd::SandboxBuilder, Crate, Toolchain, WorkspaceBuilder}; +use std::collections::HashSet; +use std::path::Path; +use utils::{get_current_versions, parse_rustc_version, resolve_deps}; +use Metadata; + +pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Result { + // TODO: Handle workspace path correctly + let rustwide_workspace = + WorkspaceBuilder::new(Path::new("/tmp/docs-builder"), "docsrs").init()?; + + // TODO: Instead of using just nightly, we can pin a version. + // Docs.rs can only use nightly (due to unstable docs.rs features in rustdoc) + let toolchain = Toolchain::Dist { + name: "nightly".into(), + }; + toolchain.install(&rustwide_workspace)?; + + let krate = Crate::crates_io(name, version); + krate.fetch(&rustwide_workspace)?; + + // Configure a sandbox with 1GB of RAM and no network access + // TODO: 1GB might not be enough + let sandbox = SandboxBuilder::new() + .memory_limit(Some(1024 * 1024 * 1024)) + .enable_networking(false); + + let mut build_dir = rustwide_workspace.build_dir(&format!("{}-{}", name, version)); + let pkg = build_dir.build(&toolchain, &krate, sandbox, |build| { + enable_nightly_features(); + let config = Config::default()?; + let source_id = try!(SourceId::crates_io(&config)); + let source_cfg_map = try!(SourceConfigMap::new(&config)); + let manifest_path = build.host_source_dir().join("Cargo.toml"); + let ws = Workspace::new(&manifest_path, &config)?; + let pkg = ws.load(&manifest_path)?; + + let metadata = Metadata::from_package(&pkg).map_err(|e| internal(e.to_string()))?; + + let mut rustdoc_flags: Vec = vec![ + "-Z".to_string(), + "unstable-options".to_string(), + "--resource-suffix".to_string(), + // FIXME: We need to get rustc version inside of container. + // Our get_current_versions gets rustc version from host system not container. + format!("-{}", parse_rustc_version(get_current_versions()?.0)?), + "--static-root-path".to_string(), + "/".to_string(), + "--disable-per-crate-search".to_string(), + ]; + + let source = try!(source_cfg_map.load(source_id, &HashSet::new())); + let _lock = try!(config.acquire_package_cache_lock()); + + for (name, dep) in try!(resolve_deps(&pkg, &config, source)) { + rustdoc_flags.push("--extern-html-root-url".to_string()); + rustdoc_flags.push(format!( + "{}=https://docs.rs/{}/{}", + name.replace("-", "_"), + dep.name(), + dep.version() + )); + } + + let mut cargo_args = vec!["doc".to_owned(), "--lib".to_owned(), "--no-deps".to_owned()]; + if let Some(features) = &metadata.features { + cargo_args.push("--features".to_owned()); + cargo_args.push(features.join(" ")); + } + if metadata.all_features { + cargo_args.push("--all-features".to_owned()); + } + if metadata.no_default_features { + cargo_args.push("--no-default-features".to_owned()); + } + + // TODO: We need to use build result here + // FIXME: We also need build log (basically stderr message) + let result = build + .cargo() + .env( + "RUSTFLAGS", + metadata + .rustc_args + .map(|args| args.join("")) + .unwrap_or("".to_owned()), + ) + .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) + .args(&cargo_args) + .run(); + + // TODO: We need to return build result as well + Ok(pkg) + })?; + + Ok(pkg) +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 6f03bc9ac..f3b24d708 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,7 +1,8 @@ //! Various utilities for cratesfyi -pub use self::build_doc::{build_doc, get_package, source_path, update_sources}; +pub use self::build_doc::{build_doc, get_package, source_path, update_sources, resolve_deps}; +pub use self::build_doc_rustwide::build_doc_rustwide; pub use self::copy::{copy_dir, copy_doc_dir}; pub use self::github_updater::github_updater; pub use self::release_activity_updater::update_release_activity; @@ -12,6 +13,7 @@ pub use self::queue::add_crate_to_queue; mod github_updater; mod build_doc; +mod build_doc_rustwide; mod copy; mod release_activity_updater; mod daemon; From bbd19a59e8fb79c6a4cf8424e6d46e506780ff7c Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 11 Sep 2019 09:47:47 +0200 Subject: [PATCH 02/23] temporarily switch to rustwide's git repository --- Cargo.lock | 18 ++++++++++-------- Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e257463f..1d3e68a2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -451,7 +451,7 @@ dependencies = [ "rusoto_credential 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", "rusoto_s3 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", - "rustwide 0.1.0 (git+https://github.com/onur/crater.git?rev=6907da8)", + "rustwide 0.2.0 (git+https://github.com/rust-lang/rustwide)", "sass-rs 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama_postgres 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2040,7 +2040,7 @@ dependencies = [ [[package]] name = "remove_dir_all" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2193,16 +2193,18 @@ dependencies = [ [[package]] name = "rustwide" -version = "0.1.0" -source = "git+https://github.com/onur/crater.git?rev=6907da8#6907da82bfcbcd74a25e24c50a834aae8a552d42" +version = "0.2.0" +source = "git+https://github.com/rust-lang/rustwide#fa1480bdedd4fd3f242e4f669839064ba088bf08" dependencies = [ "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2661,7 +2663,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2673,7 +2675,7 @@ dependencies = [ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3447,7 +3449,7 @@ dependencies = [ "checksum regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0b2f0808e7d7e4fb1cb07feb6ff2f4bc827938f24f8c2e6a3beb7370af544bdd" "checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" "checksum regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d76410686f9e3a17f06128962e0ecc5755870bb890c34820c7af7f1db2e1d48" -"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" +"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" "checksum reqwest 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1d68c7bf0b1dc3860b80c6d31d05808bf54cdc1bfc70a4680893791becd083ae" "checksum route-recognizer 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3255338088df8146ba63d60a9b8e3556f1146ce2973bc05a75181a42ce2256" "checksum router 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9b1797ff166029cb632237bb5542696e54961b4cf75a324c6f05c9cf0584e4e" @@ -3460,7 +3462,7 @@ dependencies = [ "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustfix 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "af7c21531a91512a4a51b490be6ba1c8eff34fdda0dc5bf87dc28d86748aac56" "checksum rusttype 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b8eb11f5b0a98c8eca2fb1483f42646d8c340e83e46ab416f8a063a0fd0eeb20" -"checksum rustwide 0.1.0 (git+https://github.com/onur/crater.git?rev=6907da8)" = "" +"checksum rustwide 0.2.0 (git+https://github.com/rust-lang/rustwide)" = "" "checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" diff --git a/Cargo.toml b/Cargo.toml index 28aa08da1..db8528f63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ tokio = "0.1" systemstat = "0.1.4" prometheus = { version = "0.7.0", default-features = false } lazy_static = "1.0.0" -rustwide = { git = "https://github.com/onur/crater.git", rev = "6907da8" } +rustwide = { git = "https://github.com/rust-lang/rustwide" } # iron dependencies iron = "0.5" From 8059fdc5175e606d571d3fe449105117ade4f062 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 11 Sep 2019 09:48:56 +0200 Subject: [PATCH 03/23] correctly fetch rustwide's rustc version --- src/utils/build_doc_rustwide.rs | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/utils/build_doc_rustwide.rs b/src/utils/build_doc_rustwide.rs index 87c1cb1bc..dc2e49212 100644 --- a/src/utils/build_doc_rustwide.rs +++ b/src/utils/build_doc_rustwide.rs @@ -1,11 +1,14 @@ -use cargo::core::{enable_nightly_features, Package, SourceId, Workspace}; +use cargo::core::{enable_nightly_features, Package, SourceId, Workspace as CargoWorkspace}; use cargo::sources::SourceConfigMap; use cargo::util::{internal, Config}; use error::Result; -use rustwide::{cmd::SandboxBuilder, Crate, Toolchain, WorkspaceBuilder}; +use rustwide::{ + cmd::{Command, SandboxBuilder}, + Crate, Toolchain, Workspace, WorkspaceBuilder, +}; use std::collections::HashSet; use std::path::Path; -use utils::{get_current_versions, parse_rustc_version, resolve_deps}; +use utils::{parse_rustc_version, resolve_deps}; use Metadata; pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Result { @@ -36,7 +39,7 @@ pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Re let source_id = try!(SourceId::crates_io(&config)); let source_cfg_map = try!(SourceConfigMap::new(&config)); let manifest_path = build.host_source_dir().join("Cargo.toml"); - let ws = Workspace::new(&manifest_path, &config)?; + let ws = CargoWorkspace::new(&manifest_path, &config)?; let pkg = ws.load(&manifest_path)?; let metadata = Metadata::from_package(&pkg).map_err(|e| internal(e.to_string()))?; @@ -45,9 +48,10 @@ pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Re "-Z".to_string(), "unstable-options".to_string(), "--resource-suffix".to_string(), - // FIXME: We need to get rustc version inside of container. - // Our get_current_versions gets rustc version from host system not container. - format!("-{}", parse_rustc_version(get_current_versions()?.0)?), + format!( + "-{}", + parse_rustc_version(rustc_version(&rustwide_workspace, &toolchain)?)? + ), "--static-root-path".to_string(), "/".to_string(), "--disable-per-crate-search".to_string(), @@ -99,3 +103,18 @@ pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Re Ok(pkg) } + +fn rustc_version(workspace: &Workspace, toolchain: &Toolchain) -> Result { + let res = Command::new(workspace, toolchain.rustc()) + .args(&["--version"]) + .log_output(false) + .run_capture()?; + + if let Some(line) = res.stdout_lines().iter().next() { + Ok(line.clone()) + } else { + Err(::failure::err_msg( + "invalid output returned by `rustc --version`", + )) + } +} From c11554521f6dbf5bf28c8a54b90b3ae5307bc9e3 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 11 Sep 2019 09:49:26 +0200 Subject: [PATCH 04/23] move sandbox configuration to constants --- src/utils/build_doc_rustwide.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/utils/build_doc_rustwide.rs b/src/utils/build_doc_rustwide.rs index dc2e49212..97d52ac5d 100644 --- a/src/utils/build_doc_rustwide.rs +++ b/src/utils/build_doc_rustwide.rs @@ -11,6 +11,10 @@ use std::path::Path; use utils::{parse_rustc_version, resolve_deps}; use Metadata; +// TODO: 1GB might not be enough +const SANDBOX_MEMORY_LIMIT: usize = 1024 * 1025 * 1024; // 1GB +const SANDBOX_NETWORKING: bool = false; + pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Result { // TODO: Handle workspace path correctly let rustwide_workspace = @@ -26,11 +30,9 @@ pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Re let krate = Crate::crates_io(name, version); krate.fetch(&rustwide_workspace)?; - // Configure a sandbox with 1GB of RAM and no network access - // TODO: 1GB might not be enough let sandbox = SandboxBuilder::new() - .memory_limit(Some(1024 * 1024 * 1024)) - .enable_networking(false); + .memory_limit(Some(SANDBOX_MEMORY_LIMIT)) + .enable_networking(SANDBOX_NETWORKING); let mut build_dir = rustwide_workspace.build_dir(&format!("{}-{}", name, version)); let pkg = build_dir.build(&toolchain, &krate, sandbox, |build| { From 4dd22ee450c4458913432630c427f47be0049df9 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 11 Sep 2019 10:15:47 +0200 Subject: [PATCH 05/23] build docs with rustwide on different targets --- src/utils/build_doc_rustwide.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/utils/build_doc_rustwide.rs b/src/utils/build_doc_rustwide.rs index 97d52ac5d..fcdb508c5 100644 --- a/src/utils/build_doc_rustwide.rs +++ b/src/utils/build_doc_rustwide.rs @@ -26,6 +26,9 @@ pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Re name: "nightly".into(), }; toolchain.install(&rustwide_workspace)?; + if let Some(target) = target { + toolchain.add_target(&rustwide_workspace, target)?; + } let krate = Crate::crates_io(name, version); krate.fetch(&rustwide_workspace)?; @@ -83,6 +86,10 @@ pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Re if metadata.no_default_features { cargo_args.push("--no-default-features".to_owned()); } + if let Some(target) = target { + cargo_args.push("--target".into()); + cargo_args.push(target.into()); + } // TODO: We need to use build result here // FIXME: We also need build log (basically stderr message) From 0f037c39dd2aec42f73ea3b0843e83f5cf06dd88 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 11 Sep 2019 11:07:44 +0200 Subject: [PATCH 06/23] use the cargo metadata cli instead of the library to resolve deps --- src/utils/build_doc_rustwide.rs | 42 ++++++++---- src/utils/cargo_metadata.rs | 115 ++++++++++++++++++++++++++++++++ src/utils/mod.rs | 1 + 3 files changed, 144 insertions(+), 14 deletions(-) create mode 100644 src/utils/cargo_metadata.rs diff --git a/src/utils/build_doc_rustwide.rs b/src/utils/build_doc_rustwide.rs index fcdb508c5..e0a3eb44a 100644 --- a/src/utils/build_doc_rustwide.rs +++ b/src/utils/build_doc_rustwide.rs @@ -1,21 +1,24 @@ -use cargo::core::{enable_nightly_features, Package, SourceId, Workspace as CargoWorkspace}; -use cargo::sources::SourceConfigMap; +use cargo::core::{enable_nightly_features, Workspace as CargoWorkspace}; use cargo::util::{internal, Config}; use error::Result; use rustwide::{ cmd::{Command, SandboxBuilder}, Crate, Toolchain, Workspace, WorkspaceBuilder, }; -use std::collections::HashSet; use std::path::Path; -use utils::{parse_rustc_version, resolve_deps}; +use utils::cargo_metadata::CargoMetadata; +use utils::parse_rustc_version; use Metadata; // TODO: 1GB might not be enough const SANDBOX_MEMORY_LIMIT: usize = 1024 * 1025 * 1024; // 1GB const SANDBOX_NETWORKING: bool = false; -pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Result { +pub fn build_doc_rustwide( + name: &str, + version: &str, + target: Option<&str>, +) -> Result { // TODO: Handle workspace path correctly let rustwide_workspace = WorkspaceBuilder::new(Path::new("/tmp/docs-builder"), "docsrs").init()?; @@ -41,14 +44,15 @@ pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Re let pkg = build_dir.build(&toolchain, &krate, sandbox, |build| { enable_nightly_features(); let config = Config::default()?; - let source_id = try!(SourceId::crates_io(&config)); - let source_cfg_map = try!(SourceConfigMap::new(&config)); let manifest_path = build.host_source_dir().join("Cargo.toml"); let ws = CargoWorkspace::new(&manifest_path, &config)?; let pkg = ws.load(&manifest_path)?; let metadata = Metadata::from_package(&pkg).map_err(|e| internal(e.to_string()))?; + let cargo_metadata = + CargoMetadata::load(&rustwide_workspace, &toolchain, &build.host_source_dir())?; + let mut rustdoc_flags: Vec = vec![ "-Z".to_string(), "unstable-options".to_string(), @@ -62,14 +66,11 @@ pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Re "--disable-per-crate-search".to_string(), ]; - let source = try!(source_cfg_map.load(source_id, &HashSet::new())); - let _lock = try!(config.acquire_package_cache_lock()); - - for (name, dep) in try!(resolve_deps(&pkg, &config, source)) { + for dep in &cargo_metadata.root_dependencies() { rustdoc_flags.push("--extern-html-root-url".to_string()); rustdoc_flags.push(format!( "{}=https://docs.rs/{}/{}", - name.replace("-", "_"), + dep.name().replace("-", "_"), dep.name(), dep.version() )); @@ -107,19 +108,32 @@ pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Re .run(); // TODO: We need to return build result as well - Ok(pkg) + Ok(BuildDocOutput { + package_version: cargo_metadata.root().version().to_string(), + }) })?; Ok(pkg) } +pub struct BuildDocOutput { + package_version: String, +} + +impl BuildDocOutput { + pub fn package_version(&self) -> &str { + &self.package_version + } +} + fn rustc_version(workspace: &Workspace, toolchain: &Toolchain) -> Result { let res = Command::new(workspace, toolchain.rustc()) .args(&["--version"]) .log_output(false) .run_capture()?; - if let Some(line) = res.stdout_lines().iter().next() { + let mut iter = res.stdout_lines().iter(); + if let (Some(line), None) = (iter.next(), iter.next()) { Ok(line.clone()) } else { Err(::failure::err_msg( diff --git a/src/utils/cargo_metadata.rs b/src/utils/cargo_metadata.rs new file mode 100644 index 000000000..05e7227e8 --- /dev/null +++ b/src/utils/cargo_metadata.rs @@ -0,0 +1,115 @@ +use error::Result; +use rustwide::{cmd::Command, Toolchain, Workspace}; +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +pub(crate) struct CargoMetadata { + packages: HashMap, + deps_graph: HashMap>, + root_id: String, +} + +impl CargoMetadata { + pub(crate) fn load( + workspace: &Workspace, + toolchain: &Toolchain, + source_dir: &Path, + ) -> Result { + let res = Command::new(workspace, toolchain.cargo()) + .args(&["metadata", "--format-version", "1"]) + .cd(source_dir) + .log_output(false) + .run_capture()?; + + let mut iter = res.stdout_lines().iter(); + let metadata = if let (Some(serialized), None) = (iter.next(), iter.next()) { + ::rustc_serialize::json::decode::(serialized)? + } else { + return Err(::failure::err_msg( + "invalid output returned by `cargo metadata`", + )); + }; + + // Convert from Vecs to HashMaps and HashSets to get more efficient lookups + Ok(CargoMetadata { + packages: metadata + .packages + .into_iter() + .map(|pkg| { + ( + pkg.id, + Package { + name: pkg.name, + version: pkg.version, + }, + ) + }) + .collect(), + deps_graph: metadata + .resolve + .nodes + .into_iter() + .map(|node| (node.id, node.deps.into_iter().map(|d| d.pkg).collect())) + .collect(), + root_id: metadata.resolve.root, + }) + } + + pub(crate) fn root_dependencies(&self) -> Vec<&Package> { + let ids = &self.deps_graph[&self.root_id]; + self.packages + .iter() + .filter(|(id, _pkg)| ids.contains(id.as_str())) + .map(|(_id, pkg)| pkg) + .collect() + } + + pub(crate) fn root(&self) -> &Package { + &self.packages[&self.root_id] + } +} + +pub(crate) struct Package { + name: String, + version: String, +} + +impl Package { + pub(crate) fn name(&self) -> &str { + &self.name + } + + pub(crate) fn version(&self) -> &str { + &self.version + } +} + +#[derive(RustcDecodable)] +struct DeserializedMetadata { + packages: Vec, + resolve: DeserializedResolve, +} + +#[derive(RustcDecodable)] +struct DeserializedPackage { + id: String, + name: String, + version: String, +} + +#[derive(RustcDecodable)] +struct DeserializedResolve { + root: String, + nodes: Vec, +} + +#[derive(RustcDecodable)] +struct DeserializedResolveNode { + id: String, + deps: Vec, +} + +#[derive(RustcDecodable)] +struct DeserializedResolveDep { + pkg: String, +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index f3b24d708..eb13836a1 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -11,6 +11,7 @@ pub use self::rustc_version::{parse_rustc_version, get_current_versions, command pub use self::html::extract_head_and_body; pub use self::queue::add_crate_to_queue; +mod cargo_metadata; mod github_updater; mod build_doc; mod build_doc_rustwide; From 844e2dcf32adfdb855e2104dd87ae4f3fa78f04c Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 11 Sep 2019 11:59:31 +0200 Subject: [PATCH 07/23] remove last uses of the cargo library from rustwide --- src/docbuilder/metadata.rs | 8 ++++++-- src/utils/build_doc_rustwide.rs | 27 ++++++++------------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/docbuilder/metadata.rs b/src/docbuilder/metadata.rs index e99e10cb4..0919938f3 100644 --- a/src/docbuilder/metadata.rs +++ b/src/docbuilder/metadata.rs @@ -60,10 +60,14 @@ pub struct Metadata { impl Metadata { - pub fn from_package(pkg: &Package) -> Result { + pub(crate) fn from_package(pkg: &Package) -> Result { let src_path = pkg.manifest_path().parent().ok_or_else(|| err_msg("Source path not available"))?; + Metadata::from_source_dir(src_path) + } + + pub(crate) fn from_source_dir(source_dir: &Path) -> Result { for c in ["Cargo.toml.orig", "Cargo.toml"].iter() { - let manifest_path = src_path.clone().join(c); + let manifest_path = source_dir.clone().join(c); if manifest_path.exists() { return Ok(Metadata::from_manifest(manifest_path)); } diff --git a/src/utils/build_doc_rustwide.rs b/src/utils/build_doc_rustwide.rs index e0a3eb44a..e02af23c5 100644 --- a/src/utils/build_doc_rustwide.rs +++ b/src/utils/build_doc_rustwide.rs @@ -1,5 +1,3 @@ -use cargo::core::{enable_nightly_features, Workspace as CargoWorkspace}; -use cargo::util::{internal, Config}; use error::Result; use rustwide::{ cmd::{Command, SandboxBuilder}, @@ -20,38 +18,29 @@ pub fn build_doc_rustwide( target: Option<&str>, ) -> Result { // TODO: Handle workspace path correctly - let rustwide_workspace = - WorkspaceBuilder::new(Path::new("/tmp/docs-builder"), "docsrs").init()?; + let workspace = WorkspaceBuilder::new(Path::new("/tmp/docs-builder"), "docsrs").init()?; // TODO: Instead of using just nightly, we can pin a version. // Docs.rs can only use nightly (due to unstable docs.rs features in rustdoc) let toolchain = Toolchain::Dist { name: "nightly".into(), }; - toolchain.install(&rustwide_workspace)?; + toolchain.install(&workspace)?; if let Some(target) = target { - toolchain.add_target(&rustwide_workspace, target)?; + toolchain.add_target(&workspace, target)?; } let krate = Crate::crates_io(name, version); - krate.fetch(&rustwide_workspace)?; + krate.fetch(&workspace)?; let sandbox = SandboxBuilder::new() .memory_limit(Some(SANDBOX_MEMORY_LIMIT)) .enable_networking(SANDBOX_NETWORKING); - let mut build_dir = rustwide_workspace.build_dir(&format!("{}-{}", name, version)); + let mut build_dir = workspace.build_dir(&format!("{}-{}", name, version)); let pkg = build_dir.build(&toolchain, &krate, sandbox, |build| { - enable_nightly_features(); - let config = Config::default()?; - let manifest_path = build.host_source_dir().join("Cargo.toml"); - let ws = CargoWorkspace::new(&manifest_path, &config)?; - let pkg = ws.load(&manifest_path)?; - - let metadata = Metadata::from_package(&pkg).map_err(|e| internal(e.to_string()))?; - - let cargo_metadata = - CargoMetadata::load(&rustwide_workspace, &toolchain, &build.host_source_dir())?; + let metadata = Metadata::from_source_dir(&build.host_source_dir())?; + let cargo_metadata = CargoMetadata::load(&workspace, &toolchain, &build.host_source_dir())?; let mut rustdoc_flags: Vec = vec![ "-Z".to_string(), @@ -59,7 +48,7 @@ pub fn build_doc_rustwide( "--resource-suffix".to_string(), format!( "-{}", - parse_rustc_version(rustc_version(&rustwide_workspace, &toolchain)?)? + parse_rustc_version(rustc_version(&workspace, &toolchain)?)? ), "--static-root-path".to_string(), "/".to_string(), From 5965cc898dc259a391b331382e1b7d131063a06a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 11 Sep 2019 12:14:28 +0200 Subject: [PATCH 08/23] record the build log during rustwide builds --- src/bin/cratesfyi.rs | 4 ++- src/utils/build_doc_rustwide.rs | 44 ++++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index e8f3943db..6d056b626 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -5,6 +5,7 @@ extern crate clap; extern crate log; extern crate env_logger; extern crate time; +extern crate rustwide; use std::env; @@ -308,5 +309,6 @@ fn logger_init() { record.args()) }); builder.parse(&env::var("RUST_LOG").unwrap_or("cratesfyi=info".to_owned())); - builder.init(); + + rustwide::logging::init_with(builder.build()); } diff --git a/src/utils/build_doc_rustwide.rs b/src/utils/build_doc_rustwide.rs index e02af23c5..99e7428d3 100644 --- a/src/utils/build_doc_rustwide.rs +++ b/src/utils/build_doc_rustwide.rs @@ -1,6 +1,8 @@ use error::Result; +use log::LevelFilter; use rustwide::{ cmd::{Command, SandboxBuilder}, + logging::{self, LogStorage}, Crate, Toolchain, Workspace, WorkspaceBuilder, }; use std::path::Path; @@ -9,8 +11,10 @@ use utils::parse_rustc_version; use Metadata; // TODO: 1GB might not be enough -const SANDBOX_MEMORY_LIMIT: usize = 1024 * 1025 * 1024; // 1GB +const SANDBOX_MEMORY_LIMIT: usize = 1024 * 1024 * 1024; // 1GB const SANDBOX_NETWORKING: bool = false; +const SANDBOX_MAX_LOG_SIZE: usize = 1024 * 1024; // 1MB +const SANDBOX_MAX_LOG_LINES: usize = 10_000; pub fn build_doc_rustwide( name: &str, @@ -81,24 +85,29 @@ pub fn build_doc_rustwide( cargo_args.push(target.into()); } - // TODO: We need to use build result here - // FIXME: We also need build log (basically stderr message) - let result = build - .cargo() - .env( - "RUSTFLAGS", - metadata - .rustc_args - .map(|args| args.join("")) - .unwrap_or("".to_owned()), - ) - .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) - .args(&cargo_args) - .run(); + let mut storage = LogStorage::new(LevelFilter::Info); + storage.set_max_size(SANDBOX_MAX_LOG_SIZE); + storage.set_max_lines(SANDBOX_MAX_LOG_LINES); + + logging::capture(&storage, || { + build + .cargo() + .env( + "RUSTFLAGS", + metadata + .rustc_args + .map(|args| args.join("")) + .unwrap_or("".to_owned()), + ) + .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) + .args(&cargo_args) + .run() + })?; // TODO: We need to return build result as well Ok(BuildDocOutput { package_version: cargo_metadata.root().version().to_string(), + build_log: storage.to_string(), }) })?; @@ -107,12 +116,17 @@ pub fn build_doc_rustwide( pub struct BuildDocOutput { package_version: String, + build_log: String, } impl BuildDocOutput { pub fn package_version(&self) -> &str { &self.package_version } + + pub fn build_log(&self) -> &str { + &self.build_log + } } fn rustc_version(workspace: &Workspace, toolchain: &Toolchain) -> Result { From c004725d4115e84e86985b47473b5107a1a2456a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 12 Sep 2019 10:37:24 +0200 Subject: [PATCH 09/23] extract some code from chroot_builder into docbuilder Those methods will also be used by the rustwide builder in future commits. --- src/docbuilder/chroot_builder.rs | 7 ++----- src/docbuilder/mod.rs | 11 +++++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/docbuilder/chroot_builder.rs b/src/docbuilder/chroot_builder.rs index aea0fe6c5..8e95d8e0d 100644 --- a/src/docbuilder/chroot_builder.rs +++ b/src/docbuilder/chroot_builder.rs @@ -65,10 +65,7 @@ impl DocBuilder { /// Builds package documentation in chroot environment and adds into cratesfyi database pub fn build_package(&mut self, name: &str, version: &str) -> Result { // Skip crates according to options - if (self.options.skip_if_log_exists && - self.cache.contains(&format!("{}-{}", name, version)[..])) || - (self.options.skip_if_exists && - self.db_cache.contains(&format!("{}-{}", name, version)[..])) { + if !self.should_build(name, version) { return Ok(false); } @@ -113,7 +110,7 @@ impl DocBuilder { try!(self.clean(&pkg)); // add package into build cache - self.cache.insert(format!("{}-{}", name, version)); + self.add_to_cache(name, version); Ok(res.build_success) } diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index 635fe2d55..006ccc19c 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -119,4 +119,15 @@ impl DocBuilder { pub fn options(&self) -> &DocBuilderOptions { &self.options } + + fn add_to_cache(&mut self, name: &str, version: &str) { + self.cache.insert(format!("{}-{}", name, version)); + } + + fn should_build(&self, name: &str, version: &str) -> bool { + let name = format!("{}-{}", name, version); + let local = self.options.skip_if_log_exists && self.cache.contains(&name); + let db = self.options.skip_if_exists && self.db_cache.contains(&name); + !(local || db) + } } From 9d2b6b8e930725b76692bfb51ac716670653d3c5 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 12 Sep 2019 11:55:13 +0200 Subject: [PATCH 10/23] switch add_package to use cargo metadata --- src/db/add_package.rs | 136 ++++++++++++++----------------- src/db/mod.rs | 4 +- src/docbuilder/chroot_builder.rs | 18 ++-- src/utils/build_doc_rustwide.rs | 8 +- src/utils/cargo_metadata.rs | 111 +++++++++++++++++++------ src/utils/mod.rs | 1 + 6 files changed, 165 insertions(+), 113 deletions(-) diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 7def3504a..550899078 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -1,16 +1,14 @@ use ChrootBuilderResult; use Metadata; -use utils::source_path; +use utils::MetadataPackage; use regex::Regex; use std::io::prelude::*; use std::io::BufReader; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::fs; -use cargo::core::{Package, TargetKind}; -use cargo::core::dependency::Kind; use rustc_serialize::json::{Json, ToJson}; use slug::slugify; use reqwest::Client; @@ -24,27 +22,28 @@ use failure::err_msg; /// Adds a package into database. /// /// Package must be built first. -pub fn add_package_into_database(conn: &Connection, - pkg: &Package, +pub(crate) fn add_package_into_database(conn: &Connection, + metadata_pkg: &MetadataPackage, + source_dir: &Path, res: &ChrootBuilderResult, files: Option, doc_targets: Vec) -> Result { debug!("Adding package into database"); - let crate_id = try!(initialize_package_in_database(&conn, &pkg)); - let dependencies = convert_dependencies(&pkg); - let rustdoc = get_rustdoc(&pkg).unwrap_or(None); - let readme = get_readme(&pkg).unwrap_or(None); - let (release_time, yanked, downloads) = try!(get_release_time_yanked_downloads(&pkg)); - let is_library = match pkg.targets()[0].kind() { - &TargetKind::Lib(_) => true, + let crate_id = try!(initialize_package_in_database(&conn, metadata_pkg)); + let dependencies = convert_dependencies(metadata_pkg); + let rustdoc = get_rustdoc(metadata_pkg, source_dir).unwrap_or(None); + let readme = get_readme(metadata_pkg, source_dir).unwrap_or(None); + let (release_time, yanked, downloads) = try!(get_release_time_yanked_downloads(metadata_pkg)); + let is_library = match metadata_pkg.targets[0].kind.as_slice() { + &[ref kind] if kind == "lib" => true, _ => false, }; - let metadata = Metadata::from_package(pkg)?; + let metadata = Metadata::from_source_dir(source_dir)?; let release_id: i32 = { let rows = try!(conn.query("SELECT id FROM releases WHERE crate_id = $1 AND version = $2", - &[&crate_id, &format!("{}", pkg.manifest().version())])); + &[&crate_id, &format!("{}", metadata_pkg.version)])); if rows.len() == 0 { let rows = try!(conn.query("INSERT INTO releases ( @@ -62,29 +61,29 @@ pub fn add_package_into_database(conn: &Connection, ) RETURNING id", &[&crate_id, - &format!("{}", pkg.manifest().version()), + &metadata_pkg.version, &release_time, &dependencies.to_json(), - &pkg.targets()[0].name().replace("-", "_"), + &metadata_pkg.targets[0].name.replace("-", "_"), &yanked, &res.build_success, &res.have_doc, &false, // TODO: Add test status somehow - &pkg.manifest().metadata().license, - &pkg.manifest().metadata().repository, - &pkg.manifest().metadata().homepage, - &pkg.manifest().metadata().description, + &metadata_pkg.license, + &metadata_pkg.repository, + &metadata_pkg.homepage, + &metadata_pkg.description, &rustdoc, &readme, - &pkg.manifest().metadata().authors.to_json(), - &pkg.manifest().metadata().keywords.to_json(), + &metadata_pkg.authors.to_json(), + &metadata_pkg.keywords.to_json(), &res.have_examples, &downloads, &files, &doc_targets.to_json(), &is_library, &res.rustc_version, - &pkg.manifest().metadata().documentation, + &metadata_pkg.documentation, &metadata.default_target])); // return id rows.get(0).get(0) @@ -116,38 +115,38 @@ pub fn add_package_into_database(conn: &Connection, default_target = $25 WHERE crate_id = $1 AND version = $2", &[&crate_id, - &format!("{}", pkg.manifest().version()), + &format!("{}", metadata_pkg.version), &release_time, &dependencies.to_json(), - &pkg.targets()[0].name().replace("-", "_"), + &metadata_pkg.targets[0].name.replace("-", "_"), &yanked, &res.build_success, &res.have_doc, &false, // TODO: Add test status somehow - &pkg.manifest().metadata().license, - &pkg.manifest().metadata().repository, - &pkg.manifest().metadata().homepage, - &pkg.manifest().metadata().description, + &metadata_pkg.license, + &metadata_pkg.repository, + &metadata_pkg.homepage, + &metadata_pkg.description, &rustdoc, &readme, - &pkg.manifest().metadata().authors.to_json(), - &pkg.manifest().metadata().keywords.to_json(), + &metadata_pkg.authors.to_json(), + &metadata_pkg.keywords.to_json(), &res.have_examples, &downloads, &files, &doc_targets.to_json(), &is_library, &res.rustc_version, - &pkg.manifest().metadata().documentation, + &metadata_pkg.documentation, &metadata.default_target])); rows.get(0).get(0) } }; - try!(add_keywords_into_database(&conn, &pkg, &release_id)); - try!(add_authors_into_database(&conn, &pkg, &release_id)); - try!(add_owners_into_database(&conn, &pkg, &crate_id)); + try!(add_keywords_into_database(&conn, &metadata_pkg, &release_id)); + try!(add_authors_into_database(&conn, &metadata_pkg, &release_id)); + try!(add_owners_into_database(&conn, &metadata_pkg, &crate_id)); // Update versions @@ -159,13 +158,12 @@ pub fn add_package_into_database(conn: &Connection, if let Some(versions_array) = versions.as_array_mut() { let mut found = false; for version in versions_array.clone() { - if &semver::Version::parse(version.as_string().unwrap()).unwrap() == - pkg.manifest().version() { + if version.as_string().unwrap() == metadata_pkg.version { found = true; } } if !found { - versions_array.push(format!("{}", &pkg.manifest().version()).to_json()); + versions_array.push(format!("{}", &metadata_pkg.version).to_json()); } } let _ = conn.query("UPDATE crates SET versions = $1 WHERE id = $2", @@ -177,7 +175,7 @@ pub fn add_package_into_database(conn: &Connection, /// Adds a build into database -pub fn add_build_into_database(conn: &Connection, +pub(crate) fn add_build_into_database(conn: &Connection, release_id: &i32, res: &ChrootBuilderResult) -> Result { @@ -196,13 +194,12 @@ pub fn add_build_into_database(conn: &Connection, } -fn initialize_package_in_database(conn: &Connection, pkg: &Package) -> Result { - let mut rows = try!(conn.query("SELECT id FROM crates WHERE name = $1", - &[&pkg.manifest().name().as_str()])); +fn initialize_package_in_database(conn: &Connection, pkg: &MetadataPackage) -> Result { + let mut rows = try!(conn.query("SELECT id FROM crates WHERE name = $1", &[&pkg.name])); // insert crate into database if it is not exists if rows.len() == 0 { rows = try!(conn.query("INSERT INTO crates (name) VALUES ($1) RETURNING id", - &[&pkg.manifest().name().as_str()])); + &[&pkg.name])); } Ok(rows.get(0).get(0)) } @@ -210,16 +207,12 @@ fn initialize_package_in_database(conn: &Connection, pkg: &Package) -> Result -fn convert_dependencies(pkg: &Package) -> Vec<(String, String, String)> { +fn convert_dependencies(pkg: &MetadataPackage) -> Vec<(String, String, String)> { let mut dependencies: Vec<(String, String, String)> = Vec::new(); - for dependency in pkg.manifest().dependencies() { - let name = dependency.package_name().to_string(); - let version = format!("{}", dependency.version_req()); - let kind = match dependency.kind() { - Kind::Normal => "normal", - Kind::Development => "dev", - Kind::Build => "build", - }; + for dependency in &pkg.dependencies { + let name = dependency.name.clone(); + let version = dependency.req.clone(); + let kind = dependency.kind.as_ref().map(|s| s.clone()).unwrap_or_else(|| "normal".into()); dependencies.push((name, version, kind.to_string())); } dependencies @@ -227,9 +220,8 @@ fn convert_dependencies(pkg: &Package) -> Vec<(String, String, String)> { /// Reads readme if there is any read defined in Cargo.toml of a Package -fn get_readme(pkg: &Package) -> Result> { - let readme_path = PathBuf::from(try!(source_path(&pkg).ok_or_else(|| err_msg("File not found")))) - .join(pkg.manifest().metadata().readme.clone().unwrap_or("README.md".to_owned())); +fn get_readme(pkg: &MetadataPackage, source_dir: &Path) -> Result> { + let readme_path = source_dir.join(pkg.readme.clone().unwrap_or("README.md".to_owned())); if !readme_path.exists() { return Ok(None); @@ -249,14 +241,12 @@ fn get_readme(pkg: &Package) -> Result> { } -fn get_rustdoc(pkg: &Package) -> Result> { - if let Some(src_path) = pkg.manifest().targets()[0].src_path().path() { +fn get_rustdoc(pkg: &MetadataPackage, source_dir: &Path) -> Result> { + if let Some(src_path) = &pkg.targets[0].src_path { if src_path.is_absolute() { read_rust_doc(src_path) } else { - let mut path = PathBuf::from(try!(source_path(&pkg).ok_or_else(|| err_msg("File not found")))); - path.push(src_path); - read_rust_doc(path.as_path()) + read_rust_doc(&source_dir.join(src_path)) } } else { // FIXME: should we care about metabuild targets? @@ -294,11 +284,10 @@ fn read_rust_doc(file_path: &Path) -> Result> { /// Get release_time, yanked and downloads from crates.io -fn get_release_time_yanked_downloads - (pkg: &Package) - -> Result<(Option, Option, Option)> { - let url = format!("https://crates.io/api/v1/crates/{}/versions", - pkg.manifest().name()); +fn get_release_time_yanked_downloads( + pkg: &MetadataPackage, +) -> Result<(Option, Option, Option)> { + let url = format!("https://crates.io/api/v1/crates/{}/versions", pkg.name); // FIXME: There is probably better way to do this // and so many unwraps... let client = Client::new(); @@ -321,7 +310,7 @@ fn get_release_time_yanked_downloads .and_then(|v| v.as_string()) .ok_or_else(|| err_msg("Not a JSON object"))); - if &semver::Version::parse(version_num).unwrap() == pkg.manifest().version() { + if semver::Version::parse(version_num).unwrap().to_string() == pkg.version { let release_time_raw = try!(version.get("created_at") .and_then(|c| c.as_string()) .ok_or_else(|| err_msg("Not a JSON object"))); @@ -346,8 +335,8 @@ fn get_release_time_yanked_downloads /// Adds keywords into database -fn add_keywords_into_database(conn: &Connection, pkg: &Package, release_id: &i32) -> Result<()> { - for keyword in &pkg.manifest().metadata().keywords { +fn add_keywords_into_database(conn: &Connection, pkg: &MetadataPackage, release_id: &i32) -> Result<()> { + for keyword in &pkg.keywords { let slug = slugify(&keyword); let keyword_id: i32 = { let rows = try!(conn.query("SELECT id FROM keywords WHERE slug = $1", &[&slug])); @@ -371,10 +360,10 @@ fn add_keywords_into_database(conn: &Connection, pkg: &Package, release_id: &i32 /// Adds authors into database -fn add_authors_into_database(conn: &Connection, pkg: &Package, release_id: &i32) -> Result<()> { +fn add_authors_into_database(conn: &Connection, pkg: &MetadataPackage, release_id: &i32) -> Result<()> { let author_capture_re = Regex::new("^([^><]+)<*(.*?)>*$").unwrap(); - for author in &pkg.manifest().metadata().authors { + for author in &pkg.authors { if let Some(author_captures) = author_capture_re.captures(&author[..]) { let author = author_captures.get(1).map(|m| m.as_str()).unwrap_or("").trim(); let email = author_captures.get(2).map(|m| m.as_str()).unwrap_or("").trim(); @@ -405,10 +394,9 @@ fn add_authors_into_database(conn: &Connection, pkg: &Package, release_id: &i32) /// Adds owners into database -fn add_owners_into_database(conn: &Connection, pkg: &Package, crate_id: &i32) -> Result<()> { +fn add_owners_into_database(conn: &Connection, pkg: &MetadataPackage, crate_id: &i32) -> Result<()> { // owners available in: https://crates.io/api/v1/crates/rand/owners - let owners_url = format!("https://crates.io/api/v1/crates/{}/owners", - &pkg.manifest().name()); + let owners_url = format!("https://crates.io/api/v1/crates/{}/owners", pkg.name); let client = Client::new(); let mut res = try!(client.get(&owners_url[..]) .header(ACCEPT, "application/json") diff --git a/src/db/mod.rs b/src/db/mod.rs index 7c51c9f48..a83ce902a 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,7 +1,7 @@ //! Database operations -pub use self::add_package::add_package_into_database; -pub use self::add_package::add_build_into_database; +pub(crate) use self::add_package::add_package_into_database; +pub(crate) use self::add_package::add_build_into_database; pub use self::file::add_path_into_database; pub use self::migrate::migrate; diff --git a/src/docbuilder/chroot_builder.rs b/src/docbuilder/chroot_builder.rs index 8e95d8e0d..58673535d 100644 --- a/src/docbuilder/chroot_builder.rs +++ b/src/docbuilder/chroot_builder.rs @@ -3,7 +3,7 @@ use super::DocBuilder; use super::crates::crates_from_path; use super::metadata::Metadata; use utils::{get_package, source_path, copy_doc_dir, - update_sources, parse_rustc_version, command_result}; + update_sources, parse_rustc_version, command_result, CargoMetadata}; use db::{connect_db, add_package_into_database, add_build_into_database, add_path_into_database}; use cargo::core::Package; use cargo::util::CargoResultExt; @@ -99,11 +99,17 @@ impl DocBuilder { Vec::new() }; - let release_id = try!(add_package_into_database(&conn, - &pkg, - &res, - Some(file_list), - successfully_targets)); + let src_path = pkg.manifest_path() + .parent() + .ok_or_else(|| ::failure::err_msg("Source path not available"))?; + let release_id = try!(add_package_into_database( + &conn, + &CargoMetadata::from_cargo_lib_package(&pkg).root(), + src_path, + &res, + Some(file_list), + successfully_targets + )); try!(add_build_into_database(&conn, &release_id, &res)); // remove documentation, source and build directory after we are done diff --git a/src/utils/build_doc_rustwide.rs b/src/utils/build_doc_rustwide.rs index 99e7428d3..a39e8da47 100644 --- a/src/utils/build_doc_rustwide.rs +++ b/src/utils/build_doc_rustwide.rs @@ -63,9 +63,9 @@ pub fn build_doc_rustwide( rustdoc_flags.push("--extern-html-root-url".to_string()); rustdoc_flags.push(format!( "{}=https://docs.rs/{}/{}", - dep.name().replace("-", "_"), - dep.name(), - dep.version() + dep.name.replace("-", "_"), + dep.name, + dep.version )); } @@ -106,7 +106,7 @@ pub fn build_doc_rustwide( // TODO: We need to return build result as well Ok(BuildDocOutput { - package_version: cargo_metadata.root().version().to_string(), + package_version: cargo_metadata.root().version.to_string(), build_log: storage.to_string(), }) })?; diff --git a/src/utils/cargo_metadata.rs b/src/utils/cargo_metadata.rs index 05e7227e8..a1a22faab 100644 --- a/src/utils/cargo_metadata.rs +++ b/src/utils/cargo_metadata.rs @@ -1,7 +1,8 @@ +use cargo::core::Package as CargoLibPackage; use error::Result; use rustwide::{cmd::Command, Toolchain, Workspace}; use std::collections::{HashMap, HashSet}; -use std::path::Path; +use std::path::{Path, PathBuf}; pub(crate) struct CargoMetadata { packages: HashMap, @@ -35,15 +36,7 @@ impl CargoMetadata { packages: metadata .packages .into_iter() - .map(|pkg| { - ( - pkg.id, - Package { - name: pkg.name, - version: pkg.version, - }, - ) - }) + .map(|pkg| (pkg.id.clone(), pkg)) .collect(), deps_graph: metadata .resolve @@ -55,6 +48,61 @@ impl CargoMetadata { }) } + // All of this is very hacky, but it's just needed to cleanly transition the code from cargo + // lib to cargo metadata. + pub(crate) fn from_cargo_lib_package(pkg: &CargoLibPackage) -> Self { + let id = pkg.package_id().to_string(); + CargoMetadata { + packages: ::std::iter::once(( + id.clone(), + Package { + id: id.clone(), + name: pkg.name().as_str().to_string(), + version: pkg.version().to_string(), + license: pkg.manifest().metadata().license.clone(), + repository: pkg.manifest().metadata().repository.clone(), + homepage: pkg.manifest().metadata().homepage.clone(), + description: pkg.manifest().metadata().description.clone(), + documentation: pkg.manifest().metadata().documentation.clone(), + targets: pkg + .manifest() + .targets() + .iter() + .map(|target| Target { + name: target.name().to_string(), + kind: if target.is_lib() { + vec!["lib".into()] + } else { + vec![] + }, + src_path: target.src_path().path().map(|p| p.into()), + }) + .collect(), + dependencies: pkg + .manifest() + .dependencies() + .iter() + .map(|dep| Dependency { + name: dep.package_name().to_string(), + req: dep.version_req().to_string(), + kind: match dep.kind() { + ::cargo::core::dependency::Kind::Normal => None, + ::cargo::core::dependency::Kind::Development => Some("dev".into()), + ::cargo::core::dependency::Kind::Build => Some("build".into()), + }, + }) + .collect(), + readme: pkg.manifest().metadata().readme.clone(), + keywords: pkg.manifest().metadata().keywords.clone(), + authors: pkg.manifest().metadata().authors.clone(), + }, + )) + .collect(), + deps_graph: HashMap::new(), + root_id: id, + } + } + pub(crate) fn root_dependencies(&self) -> Vec<&Package> { let ids = &self.deps_graph[&self.root_id]; self.packages @@ -69,32 +117,41 @@ impl CargoMetadata { } } +#[derive(RustcDecodable)] pub(crate) struct Package { - name: String, - version: String, + pub(crate) id: String, + pub(crate) name: String, + pub(crate) version: String, + pub(crate) license: Option, + pub(crate) repository: Option, + pub(crate) homepage: Option, + pub(crate) description: Option, + pub(crate) documentation: Option, + pub(crate) dependencies: Vec, + pub(crate) targets: Vec, + pub(crate) readme: Option, + pub(crate) keywords: Vec, + pub(crate) authors: Vec, } -impl Package { - pub(crate) fn name(&self) -> &str { - &self.name - } - - pub(crate) fn version(&self) -> &str { - &self.version - } +#[derive(RustcDecodable)] +pub(crate) struct Target { + pub(crate) name: String, + pub(crate) kind: Vec, + pub(crate) src_path: Option, } #[derive(RustcDecodable)] -struct DeserializedMetadata { - packages: Vec, - resolve: DeserializedResolve, +pub(crate) struct Dependency { + pub(crate) name: String, + pub(crate) req: String, + pub(crate) kind: Option, } #[derive(RustcDecodable)] -struct DeserializedPackage { - id: String, - name: String, - version: String, +struct DeserializedMetadata { + packages: Vec, + resolve: DeserializedResolve, } #[derive(RustcDecodable)] diff --git a/src/utils/mod.rs b/src/utils/mod.rs index eb13836a1..96a81aa6c 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -10,6 +10,7 @@ pub use self::daemon::start_daemon; pub use self::rustc_version::{parse_rustc_version, get_current_versions, command_result}; pub use self::html::extract_head_and_body; pub use self::queue::add_crate_to_queue; +pub(crate) use self::cargo_metadata::{CargoMetadata, Package as MetadataPackage}; mod cargo_metadata; mod github_updater; From f23d0c1d884fc7115522ef31b3ab75a2367fb430 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 12 Sep 2019 18:53:07 +0200 Subject: [PATCH 11/23] replace chroot_builder with rustwide_builder --- .gitignore | 1 + Cargo.toml | 4 +- src/bin/cratesfyi.rs | 62 +--- src/db/add_package.rs | 29 +- src/docbuilder/chroot_builder.rs | 542 ----------------------------- src/docbuilder/metadata.rs | 6 - src/docbuilder/mod.rs | 5 +- src/docbuilder/queue.rs | 10 +- src/docbuilder/rustwide_builder.rs | 439 +++++++++++++++++++++++ src/lib.rs | 3 +- src/utils/build_doc.rs | 245 ------------- src/utils/build_doc_rustwide.rs | 146 -------- src/utils/cargo_metadata.rs | 60 +--- src/utils/copy.rs | 1 - src/utils/daemon.rs | 19 +- src/utils/mod.rs | 4 - 16 files changed, 487 insertions(+), 1089 deletions(-) delete mode 100644 src/docbuilder/chroot_builder.rs create mode 100644 src/docbuilder/rustwide_builder.rs delete mode 100644 src/utils/build_doc.rs delete mode 100644 src/utils/build_doc_rustwide.rs diff --git a/.gitignore b/.gitignore index 613083218..23a5c4c78 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ target *.css.map .sass-cache .vagrant +/rustwide diff --git a/Cargo.toml b/Cargo.toml index db8528f63..563550bd6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ systemstat = "0.1.4" prometheus = { version = "0.7.0", default-features = false } lazy_static = "1.0.0" rustwide = { git = "https://github.com/rust-lang/rustwide" } +tempdir = "0.3" # iron dependencies iron = "0.5" @@ -52,9 +53,6 @@ staticfile = { version = "0.4", features = [ "cache" ] } version = "0.15" features = [ "with-time", "with-rustc-serialize" ] -[dev-dependencies] -tempdir = "0.3" - [build-dependencies] time = "0.1" git2 = "0.9" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 6d056b626..34a28a47c 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -9,11 +9,11 @@ extern crate rustwide; use std::env; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use clap::{Arg, App, SubCommand}; -use cratesfyi::{DocBuilder, DocBuilderOptions, db}; -use cratesfyi::utils::{build_doc, build_doc_rustwide, add_crate_to_queue}; +use cratesfyi::{DocBuilder, RustwideBuilder, DocBuilderOptions, db}; +use cratesfyi::utils::add_crate_to_queue; use cratesfyi::start_web_server; use cratesfyi::db::{add_path_into_database, connect_db}; @@ -24,34 +24,6 @@ pub fn main() { let matches = App::new("cratesfyi") .version(cratesfyi::BUILD_VERSION) .about(env!("CARGO_PKG_DESCRIPTION")) - .subcommand(SubCommand::with_name("doc") - .about("Builds documentation of a crate") - .arg(Arg::with_name("CRATE_NAME") - .index(1) - .required(true) - .help("Crate name")) - .arg(Arg::with_name("CRATE_VERSION") - .index(2) - .required(false) - .help("Crate version")) - .arg(Arg::with_name("TARGET") - .index(3) - .required(false) - .help("The target platform to compile for"))) - .subcommand(SubCommand::with_name("doc_rustwide") - .about("Builds documentation of a crate with rustwide") - .arg(Arg::with_name("CRATE_NAME") - .index(1) - .required(true) - .help("Crate name")) - .arg(Arg::with_name("CRATE_VERSION") - .index(2) - .required(true) - .help("Crate version")) - .arg(Arg::with_name("TARGET") - .index(3) - .required(false) - .help("The target platform to compile for"))) .subcommand(SubCommand::with_name("build") .about("Builds documentation in a chroot environment") .arg(Arg::with_name("PREFIX") @@ -167,22 +139,7 @@ pub fn main() { - // doc subcommand - if let Some(matches) = matches.subcommand_matches("doc") { - let name = matches.value_of("CRATE_NAME").unwrap(); - let version = matches.value_of("CRATE_VERSION"); - let target = matches.value_of("TARGET"); - if let Err(e) = build_doc(name, version, target) { - panic!("{:#?}", e); - } - } else if let Some(matches) = matches.subcommand_matches("doc_rustwide") { - let name = matches.value_of("CRATE_NAME").unwrap(); - let version = matches.value_of("CRATE_VERSION").unwrap(); - let target = matches.value_of("TARGET"); - if let Err(e) = build_doc_rustwide(name, version, target) { - panic!("{:#?}", e); - } - } else if let Some(matches) = matches.subcommand_matches("build") { + if let Some(matches) = matches.subcommand_matches("build") { let docbuilder_opts = { let mut docbuilder_opts = if let Some(prefix) = matches.value_of("PREFIX") { DocBuilderOptions::from_prefix(PathBuf::from(prefix)) @@ -226,16 +183,19 @@ pub fn main() { if let Some(_) = matches.subcommand_matches("world") { docbuilder.load_cache().expect("Failed to load cache"); - docbuilder.build_world().expect("Failed to build world"); + let mut builder = RustwideBuilder::init(&Path::new("rustwide")).unwrap(); + builder.build_world(&mut docbuilder).expect("Failed to build world"); docbuilder.save_cache().expect("Failed to save cache"); } else if let Some(matches) = matches.subcommand_matches("crate") { docbuilder.load_cache().expect("Failed to load cache"); - docbuilder.build_package(matches.value_of("CRATE_NAME").unwrap(), - matches.value_of("CRATE_VERSION").unwrap()) + let mut builder = RustwideBuilder::init(&Path::new("rustwide")).unwrap(); + builder.build_package(&mut docbuilder, matches.value_of("CRATE_NAME").unwrap(), + matches.value_of("CRATE_VERSION").unwrap()) .expect("Building documentation failed"); docbuilder.save_cache().expect("Failed to save cache"); } else if let Some(_) = matches.subcommand_matches("add-essential-files") { - docbuilder.add_essential_files().expect("Failed to add essential files"); + let builder = RustwideBuilder::init(&Path::new("rustwide")).unwrap(); + builder.add_essential_files().expect("failed to add essential files"); } else if let Some(_) = matches.subcommand_matches("lock") { docbuilder.lock().expect("Failed to lock"); } else if let Some(_) = matches.subcommand_matches("unlock") { diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 550899078..6f06daa93 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -1,7 +1,7 @@ -use ChrootBuilderResult; use Metadata; use utils::MetadataPackage; +use docbuilder::BuildResult; use regex::Regex; use std::io::prelude::*; @@ -25,9 +25,11 @@ use failure::err_msg; pub(crate) fn add_package_into_database(conn: &Connection, metadata_pkg: &MetadataPackage, source_dir: &Path, - res: &ChrootBuilderResult, + res: &BuildResult, files: Option, - doc_targets: Vec) + doc_targets: Vec, + has_docs: bool, + has_examples: bool) -> Result { debug!("Adding package into database"); let crate_id = try!(initialize_package_in_database(&conn, metadata_pkg)); @@ -66,8 +68,8 @@ pub(crate) fn add_package_into_database(conn: &Connection, &dependencies.to_json(), &metadata_pkg.targets[0].name.replace("-", "_"), &yanked, - &res.build_success, - &res.have_doc, + &res.successful, + &has_docs, &false, // TODO: Add test status somehow &metadata_pkg.license, &metadata_pkg.repository, @@ -77,7 +79,7 @@ pub(crate) fn add_package_into_database(conn: &Connection, &readme, &metadata_pkg.authors.to_json(), &metadata_pkg.keywords.to_json(), - &res.have_examples, + &has_examples, &downloads, &files, &doc_targets.to_json(), @@ -120,8 +122,8 @@ pub(crate) fn add_package_into_database(conn: &Connection, &dependencies.to_json(), &metadata_pkg.targets[0].name.replace("-", "_"), &yanked, - &res.build_success, - &res.have_doc, + &res.successful, + &has_docs, &false, // TODO: Add test status somehow &metadata_pkg.license, &metadata_pkg.repository, @@ -131,7 +133,7 @@ pub(crate) fn add_package_into_database(conn: &Connection, &readme, &metadata_pkg.authors.to_json(), &metadata_pkg.keywords.to_json(), - &res.have_examples, + &has_examples, &downloads, &files, &doc_targets.to_json(), @@ -177,7 +179,7 @@ pub(crate) fn add_package_into_database(conn: &Connection, /// Adds a build into database pub(crate) fn add_build_into_database(conn: &Connection, release_id: &i32, - res: &ChrootBuilderResult) + res: &BuildResult) -> Result { debug!("Adding build into database"); let rows = try!(conn.query("INSERT INTO builds (rid, rustc_version, @@ -187,9 +189,9 @@ pub(crate) fn add_build_into_database(conn: &Connection, RETURNING id", &[release_id, &res.rustc_version, - &res.cratesfyi_version, - &res.build_success, - &res.output])); + &res.docsrs_version, + &res.successful, + &res.build_log])); Ok(rows.get(0).get(0)) } @@ -243,6 +245,7 @@ fn get_readme(pkg: &MetadataPackage, source_dir: &Path) -> Result fn get_rustdoc(pkg: &MetadataPackage, source_dir: &Path) -> Result> { if let Some(src_path) = &pkg.targets[0].src_path { + let src_path = Path::new(src_path); if src_path.is_absolute() { read_rust_doc(src_path) } else { diff --git a/src/docbuilder/chroot_builder.rs b/src/docbuilder/chroot_builder.rs deleted file mode 100644 index 58673535d..000000000 --- a/src/docbuilder/chroot_builder.rs +++ /dev/null @@ -1,542 +0,0 @@ - -use super::DocBuilder; -use super::crates::crates_from_path; -use super::metadata::Metadata; -use utils::{get_package, source_path, copy_doc_dir, - update_sources, parse_rustc_version, command_result, CargoMetadata}; -use db::{connect_db, add_package_into_database, add_build_into_database, add_path_into_database}; -use cargo::core::Package; -use cargo::util::CargoResultExt; -use std::process::Command; -use std::path::{Path, PathBuf}; -use std::fs::{self, remove_dir_all}; -use postgres::Connection; -use rustc_serialize::json::{Json, ToJson}; -use error::Result; -use systemstat::{Platform, Filesystem, System}; - -const MAX_DISK_USAGE: f32 = 80.0; - -/// List of targets supported by docs.rs -const TARGETS: [&'static str; 6] = [ - "i686-apple-darwin", - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu" -]; - - - -#[derive(Debug)] -pub struct ChrootBuilderResult { - pub output: String, - pub build_success: bool, - pub have_doc: bool, - pub have_examples: bool, - pub rustc_version: String, - pub cratesfyi_version: String, -} - - -impl DocBuilder { - /// Builds every package documentation in chroot environment - pub fn build_world(&mut self) -> Result<()> { - try!(update_sources()); - - let mut count = 0; - - crates(self.options.crates_io_index_path.clone(), |name, version| { - match self.build_package(name, version) { - Ok(status) => { - count += 1; - if status && count % 10 == 0 { - let _ = self.save_cache(); - } - } - Err(err) => warn!("Failed to build package {}-{}: {}", name, version, err), - } - self.cache.insert(format!("{}-{}", name, version)); - }) - } - - - /// Builds package documentation in chroot environment and adds into cratesfyi database - pub fn build_package(&mut self, name: &str, version: &str) -> Result { - // Skip crates according to options - if !self.should_build(name, version) { - return Ok(false); - } - - info!("Building package {}-{}", name, version); - - // Start with clean documentation directory - try!(self.remove_build_dir()); - - // Database connection - let conn = try!(connect_db()); - - // get_package (and cargo) is using semver, add '=' in front of version. - let pkg = try!(get_package(name, Some(&format!("={}", version)[..]))); - let metadata = Metadata::from_package(&pkg)?; - let res = self.build_package_in_chroot(&pkg, metadata.default_target.clone()); - - // copy sources and documentation - let file_list = try!(self.add_sources_into_database(&conn, &pkg)); - let successfully_targets = if res.have_doc { - try!(self.copy_documentation(&pkg, - &res.rustc_version, - metadata.default_target.as_ref().map(String::as_str), - true)); - let successfully_targets = self.build_package_for_all_targets(&pkg); - for target in &successfully_targets { - try!(self.copy_documentation(&pkg, &res.rustc_version, Some(target), false)); - } - try!(self.add_documentation_into_database(&conn, &pkg)); - successfully_targets - } else { - Vec::new() - }; - - let src_path = pkg.manifest_path() - .parent() - .ok_or_else(|| ::failure::err_msg("Source path not available"))?; - let release_id = try!(add_package_into_database( - &conn, - &CargoMetadata::from_cargo_lib_package(&pkg).root(), - src_path, - &res, - Some(file_list), - successfully_targets - )); - try!(add_build_into_database(&conn, &release_id, &res)); - - // remove documentation, source and build directory after we are done - try!(self.clean(&pkg)); - - // add package into build cache - self.add_to_cache(name, version); - - Ok(res.build_success) - } - - - /// Builds documentation of a package with cratesfyi in chroot environment - fn build_package_in_chroot(&self, package: &Package, default_target: Option) -> ChrootBuilderResult { - debug!("Building package in chroot"); - let (rustc_version, cratesfyi_version) = self.get_versions(); - let cmd = format!("cratesfyi doc {} ={} {}", - package.manifest().name(), - package.manifest().version(), - default_target.as_ref().unwrap_or(&"".to_string())); - match self.chroot_command(cmd) { - Ok(o) => { - ChrootBuilderResult { - output: o, - build_success: true, - have_doc: self.have_documentation(&package, default_target), - have_examples: self.have_examples(&package), - rustc_version: rustc_version, - cratesfyi_version: cratesfyi_version, - } - } - Err(e) => { - ChrootBuilderResult { - output: e.to_string(), - build_success: false, - have_doc: false, - have_examples: self.have_examples(&package), - rustc_version: rustc_version, - cratesfyi_version: cratesfyi_version, - } - } - } - } - - - - /// Builds documentation of crate for every target and returns Vec of successfully targets - fn build_package_for_all_targets(&self, package: &Package) -> Vec { - let mut successfuly_targets = Vec::new(); - - for target in TARGETS.iter() { - debug!("Building {} for {}", canonical_name(&package), target); - let cmd = format!("cratesfyi doc {} ={} {}", - package.manifest().name(), - package.manifest().version(), - target); - if let Ok(_) = self.chroot_command(cmd) { - // Cargo is not giving any error and not generating documentation of some crates - // when we use a target compile options. Check documentation exists before - // adding target to successfully_targets. - // FIXME: Need to figure out why some docs are not generated with target option - let target_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi") - .join(&target) - .join("doc"); - if target_doc_path.exists() { - successfuly_targets.push(target.to_string()); - } - } - } - successfuly_targets - } - - - /// Copies documentation to destination directory - fn copy_documentation(&self, - package: &Package, - rustc_version: &str, - target: Option<&str>, - is_default_target: bool) - -> Result<()> { - let mut crate_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi"); - - // docs are available in cratesfyi/$TARGET when target is being used - if let Some(target) = target { - crate_doc_path.push(target); - } - - let mut destination = PathBuf::from(&self.options.destination) - .join(format!("{}/{}", - package.manifest().name(), - package.manifest().version())); - - // only add target name to destination directory when we are copying a non-default target. - // this is allowing us to host documents in the root of the crate documentation directory. - // for example winapi will be available in docs.rs/winapi/$version/winapi/ for it's - // default target: x86_64-pc-windows-msvc. But since it will be built under - // cratesfyi/x86_64-pc-windows-msvc we still need target in this function. - if !is_default_target { - if let Some(target) = target { - destination.push(target); - } - } - - copy_doc_dir(crate_doc_path, - destination, - parse_rustc_version(rustc_version)?.trim()) - } - - - /// Removes build directory of a package in chroot - fn remove_build_dir(&self) -> Result<()> { - let crate_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi") - .join("doc"); - let _ = remove_dir_all(crate_doc_path); - for target in TARGETS.iter() { - let crate_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi") - .join(target) - .join("doc"); - let _ = remove_dir_all(crate_doc_path); - } - - let crate_build_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi"); - let fs = mount_for_path(&crate_build_path); - let disk_usage = 100.0 - 100.0 * (fs.free.as_usize() as f32 / fs.total.as_usize() as f32); - if disk_usage >= MAX_DISK_USAGE { - info!("Cleaning target directory, disk usage {:.2} exceeded {:.2}", - disk_usage, MAX_DISK_USAGE); - let _ = remove_dir_all(&crate_build_path); - let _ = fs::create_dir_all(&crate_build_path); - } - - Ok(()) - } - - - /// Remove documentation, build directory and sources directory of a package - fn clean(&self, package: &Package) -> Result<()> { - debug!("Cleaning package"); - let documentation_path = PathBuf::from(&self.options.destination) - .join(package.manifest().name().as_str()); - let source_path = source_path(&package).unwrap(); - // Some crates don't have documentation, so we don't care if removing_dir_all fails - let _ = self.remove_build_dir(); - let _ = remove_dir_all(documentation_path); - let _ = remove_dir_all(source_path); - Ok(()) - } - - - /// Runs a command in a chroot environment - fn chroot_command>(&self, cmd: T) -> Result { - command_result(Command::new("sudo") - .arg("lxc-attach") - .arg("-n") - .arg(&self.options.container_name) - .arg("--") - .arg("su") - .arg("-") - .arg(&self.options.chroot_user) - .arg("-c") - .arg(cmd.as_ref()) - .output() - .unwrap()) - } - - - /// Checks a package build directory to determine if package have docs - /// - /// This function is checking first target in targets to see if documentation exists for a - /// crate. Package must be successfully built in chroot environment first. - fn have_documentation(&self, package: &Package, default_target: Option) -> bool { - let mut crate_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi"); - - if let Some(default_doc_path) = default_target { - crate_doc_path.push(default_doc_path); - } - - crate_doc_path.push("doc"); - crate_doc_path.push(package.targets()[0].name().replace("-", "_").to_string()); - crate_doc_path.exists() - } - - - /// Checks if package have examples - fn have_examples(&self, package: &Package) -> bool { - let path = source_path(&package).unwrap().join("examples"); - path.exists() && path.is_dir() - } - - - /// Gets rustc and cratesfyi version from chroot environment - pub fn get_versions(&self) -> (String, String) { - // It is safe to use expect here - // chroot environment must always have rustc and cratesfyi installed - (String::from(self.chroot_command("rustc --version") - .expect("Failed to get rustc version") - .trim()), - String::from(self.chroot_command("cratesfyi --version") - .expect("Failed to get cratesfyi version") - .trim())) - } - - - /// Adds sources into database - fn add_sources_into_database(&self, conn: &Connection, package: &Package) -> Result { - debug!("Adding sources into database"); - let prefix = format!("sources/{}/{}", - package.manifest().name(), - package.manifest().version()); - add_path_into_database(conn, &prefix, source_path(&package).unwrap()) - } - - - /// Adds documentations into database - fn add_documentation_into_database(&self, - conn: &Connection, - package: &Package) - -> Result { - debug!("Adding documentation into database"); - let prefix = format!("rustdoc/{}/{}", - package.manifest().name(), - package.manifest().version()); - let crate_doc_path = PathBuf::from(&self.options.destination).join(format!("{}/{}", - package.manifest().name(), - package.manifest().version())); - add_path_into_database(conn, &prefix, crate_doc_path) - } - - - /// This function will build an empty crate and will add essential documentation files. - /// - /// It is required to run after every rustc update. cratesfyi is not keeping this files - /// for every crate to avoid duplications. - /// - /// List of the files: - /// - /// * rustdoc.css (with rustc version) - /// * main.css (with rustc version) - /// * main.js (with rustc version) - /// * jquery.js (with rustc version) - /// * playpen.js (with rustc version) - /// * normalize.css - /// * FiraSans-Medium.woff - /// * FiraSans-Regular.woff - /// * Heuristica-Italic.woff - /// * SourceCodePro-Regular.woff - /// * SourceCodePro-Semibold.woff - /// * SourceSerifPro-Bold.woff - /// * SourceSerifPro-Regular.woff - pub fn add_essential_files(&self) -> Result<()> { - use std::fs::{copy, create_dir_all}; - - // acme-client-0.0.0 is an empty library crate and it will always build - let pkg = try!(get_package("acme-client", Some("=0.0.0"))); - let res = self.build_package_in_chroot(&pkg, None); - let rustc_version = parse_rustc_version(&res.rustc_version)?; - - if !res.build_success { - return Err(format_err!("Failed to build empty crate for: {}", res.rustc_version)); - } - - info!("Copying essential files for: {}", res.rustc_version); - - let files = (// files require rustc version subfix - ["brush.svg", - "wheel.svg", - "down-arrow.svg", - "dark.css", - "light.css", - "main.js", - "normalize.css", - "rustdoc.css", - "settings.css", - "settings.js", - "storage.js", - "theme.js", - "source-script.js", - "noscript.css", - "rust-logo.png"], - // favicon.ico is not needed because we set our own - // files doesn't require rustc version subfix - ["FiraSans-Medium.woff", - "FiraSans-Regular.woff", - "SourceCodePro-Regular.woff", - "SourceCodePro-Semibold.woff", - "SourceSerifPro-Bold.ttf.woff", - "SourceSerifPro-Regular.ttf.woff", - "SourceSerifPro-It.ttf.woff"]); - - let source = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi") - .join("doc"); - - // use copy_documentation destination directory so self.clean can remove it when - // we are done - let destination = PathBuf::from(&self.options.destination) - .join(format!("{}/{}", pkg.manifest().name(), pkg.manifest().version())); - try!(create_dir_all(&destination)); - - for file in files.0.iter() { - let spl: Vec<&str> = file.split('.').collect(); - let file_name = format!("{}-{}.{}", spl[0], rustc_version, spl[1]); - let source_path = source.join(&file_name); - let destination_path = destination.join(&file_name); - try!(copy(&source_path, &destination_path) - .chain_err(|| format!("couldn't copy '{}' to '{}'", source_path.display(), destination_path.display()))); - } - - for file in files.1.iter() { - let source_path = source.join(file); - let destination_path = destination.join(file); - try!(copy(&source_path, &destination_path) - .chain_err(|| format!("couldn't copy '{}' to '{}'", source_path.display(), destination_path.display()))); - } - - let conn = try!(connect_db()); - try!(add_path_into_database(&conn, "", destination)); - - try!(self.clean(&pkg)); - - let (vers, _) = self.get_versions(); - - try!(conn.query("INSERT INTO config (name, value) VALUES ('rustc_version', $1)", - &[&vers.to_json()]) - .or_else(|_| { - conn.query("UPDATE config SET value = $1 WHERE name = 'rustc_version'", - &[&vers.to_json()]) - })); - - Ok(()) - } -} - - -/// Returns canonical name of a package. -/// -/// It's just package-version. All directory structure used in cratesfyi is -/// following this naming scheme. -fn canonical_name(package: &Package) -> String { - format!("{}-{}", - package.manifest().name(), - package.manifest().version()) -} - - -/// Runs `func` with the all crates from crates-io.index repository path. -/// -/// First argument of func is the name of crate and -/// second argument is the version of crate. Func will be run for every crate. -fn crates(path: PathBuf, mut func: F) -> Result<()> - where F: FnMut(&str, &str) -> () -{ - crates_from_path(&path, &mut func) -} - -fn mount_for_path(path: &Path) -> Filesystem { - let system = System::new(); - - let mut found = None; - let mut found_pos = std::usize::MAX; - for mount in system.mounts().expect("get all mounts").into_iter() { - let mount_path = Path::new(&mount.fs_mounted_on); - for (i, ancestor) in path.ancestors().enumerate() { - if ancestor == mount_path && i < found_pos { - found_pos = i; - found = Some(mount); - break; - } - } - } - found.expect("on a disk mount") -} - - -#[cfg(test)] -mod test { - extern crate env_logger; - use std::path::PathBuf; - use {DocBuilder, DocBuilderOptions}; - - #[test] - #[ignore] - fn test_build_world() { - let _ = env_logger::try_init(); - let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix")); - let mut docbuilder = DocBuilder::new(options); - // This test is building WHOLE WORLD and may take forever - assert!(docbuilder.build_world().is_ok()); - } - - #[test] - #[ignore] - fn test_build_package() { - let _ = env_logger::try_init(); - let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix")); - let mut docbuilder = DocBuilder::new(options); - let res = docbuilder.build_package("rand", "0.3.14"); - assert!(res.is_ok()); - } - - #[test] - #[ignore] - fn test_add_essential_files() { - let _ = env_logger::try_init(); - let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix")); - let docbuilder = DocBuilder::new(options); - - docbuilder.add_essential_files().unwrap(); - } -} diff --git a/src/docbuilder/metadata.rs b/src/docbuilder/metadata.rs index 0919938f3..5fdcd41b9 100644 --- a/src/docbuilder/metadata.rs +++ b/src/docbuilder/metadata.rs @@ -1,6 +1,5 @@ use std::path::Path; -use cargo::core::Package; use toml::Value; use error::Result; use failure::err_msg; @@ -60,11 +59,6 @@ pub struct Metadata { impl Metadata { - pub(crate) fn from_package(pkg: &Package) -> Result { - let src_path = pkg.manifest_path().parent().ok_or_else(|| err_msg("Source path not available"))?; - Metadata::from_source_dir(src_path) - } - pub(crate) fn from_source_dir(source_dir: &Path) -> Result { for c in ["Cargo.toml.orig", "Cargo.toml"].iter() { let manifest_path = source_dir.clone().join(c); diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index 006ccc19c..d4bdca33d 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -1,11 +1,12 @@ pub mod options; pub mod metadata; -mod chroot_builder; +mod rustwide_builder; mod crates; mod queue; -pub use self::chroot_builder::ChrootBuilderResult; +pub use self::rustwide_builder::RustwideBuilder; +pub(crate) use self::rustwide_builder::BuildResult; use std::fs; diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index ae07e0a5e..f6448b489 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -1,7 +1,6 @@ //! Updates crates.io index and builds new packages - -use super::DocBuilder; +use super::{DocBuilder, RustwideBuilder}; use db::connect_db; use error::Result; use crates_index_diff::{ChangeKind, Index}; @@ -38,7 +37,10 @@ impl DocBuilder { } /// Builds the top package from the queue. Returns whether the queue was empty. - pub fn build_next_queue_package(&mut self) -> Result { + pub(crate) fn build_next_queue_package( + &mut self, + builder: &mut RustwideBuilder, + ) -> Result { let conn = try!(connect_db()); let query = try!(conn.query("SELECT id, name, version @@ -57,7 +59,7 @@ impl DocBuilder { let name: String = query.get(0).get(1); let version: String = query.get(0).get(2); - match self.build_package(&name[..], &version[..]) { + match builder.build_package(self, &name, &version) { Ok(_) => { let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]); } diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs new file mode 100644 index 000000000..59e088412 --- /dev/null +++ b/src/docbuilder/rustwide_builder.rs @@ -0,0 +1,439 @@ +use super::DocBuilder; +use db::file::add_path_into_database; +use db::{add_build_into_database, add_package_into_database, connect_db}; +use docbuilder::crates::crates_from_path; +use error::Result; +use failure::ResultExt; +use log::LevelFilter; +use postgres::Connection; +use rustc_serialize::json::ToJson; +use rustwide::cmd::{Command, SandboxBuilder}; +use rustwide::logging::{self, LogStorage}; +use rustwide::{Build, Crate, Toolchain, Workspace, WorkspaceBuilder}; +use std::path::Path; +use std::time::Duration; +use utils::{copy_doc_dir, parse_rustc_version, CargoMetadata}; +use Metadata; + +// TODO: 1GB might not be enough +const SANDBOX_MEMORY_LIMIT: usize = 1024 * 1024 * 1024; // 1GB +const SANDBOX_NETWORKING: bool = false; +const SANDBOX_MAX_LOG_SIZE: usize = 1024 * 1024; // 1MB +const SANDBOX_MAX_LOG_LINES: usize = 10_000; +const COMMAND_TIMEOUT: Option = Some(Duration::from_secs(60 * 60)); // 1 hour +const COMMAND_NO_OUTPUT_TIMEOUT: Option = None; + +static USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; +static TARGETS: &[&str] = &[ + "i686-apple-darwin", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", +]; +static DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; + +static ESSENTIAL_FILES_VERSIONED: &[&str] = &[ + "brush.svg", + "wheel.svg", + "down-arrow.svg", + "dark.css", + "light.css", + "main.js", + "normalize.css", + "rustdoc.css", + "settings.css", + "settings.js", + "storage.js", + "theme.js", + "source-script.js", + "noscript.css", + "rust-logo.png", +]; +static ESSENTIAL_FILES_UNVERSIONED: &[&str] = &[ + "FiraSans-Medium.woff", + "FiraSans-Regular.woff", + "SourceCodePro-Regular.woff", + "SourceCodePro-Semibold.woff", + "SourceSerifPro-Bold.ttf.woff", + "SourceSerifPro-Regular.ttf.woff", + "SourceSerifPro-It.ttf.woff", +]; + +pub struct RustwideBuilder { + workspace: Workspace, + toolchain: Toolchain, + rustc_version: String, +} + +impl RustwideBuilder { + pub fn init(workspace_path: &Path) -> Result { + let workspace = WorkspaceBuilder::new(workspace_path, USER_AGENT) + .command_timeout(COMMAND_TIMEOUT) + .command_no_output_timeout(COMMAND_NO_OUTPUT_TIMEOUT) + .init()?; + + let toolchain = Toolchain::Dist { + name: "nightly".into(), + }; + + let mut builder = RustwideBuilder { + workspace, + toolchain, + rustc_version: String::new(), + }; + builder.update_toolchain()?; + Ok(builder) + } + + fn update_toolchain(&mut self) -> Result<()> { + // Ignore errors if detection fails. + let old_version = self.detect_rustc_version().ok(); + + self.toolchain.install(&self.workspace)?; + for target in TARGETS { + self.toolchain.add_target(&self.workspace, target)?; + } + self.rustc_version = self.detect_rustc_version()?; + + if old_version.as_ref().map(|s| s.as_str()) != Some(&self.rustc_version) { + self.add_essential_files()?; + } + + Ok(()) + } + + fn detect_rustc_version(&self) -> Result { + info!("detecting rustc's version..."); + let res = Command::new(&self.workspace, self.toolchain.rustc()) + .args(&["--version"]) + .log_output(false) + .run_capture()?; + let mut iter = res.stdout_lines().iter(); + if let (Some(line), None) = (iter.next(), iter.next()) { + info!("found rustc {}", line); + Ok(line.clone()) + } else { + Err(::failure::err_msg( + "invalid output returned by `rustc --version`", + )) + } + } + + pub fn add_essential_files(&self) -> Result<()> { + info!("building a dummy crate to get essential files"); + let rustc_version = parse_rustc_version(&self.rustc_version)?; + + let mut build_dir = self + .workspace + .build_dir(&format!("essential-files-{}", rustc_version)); + build_dir.purge()?; + + // acme-client-0.0.0 is an empty library crate and it will always build + let krate = Crate::crates_io("acme-client", "0.0.0"); + krate.fetch(&self.workspace)?; + + let sandbox = SandboxBuilder::new() + .memory_limit(Some(SANDBOX_MEMORY_LIMIT)) + .enable_networking(SANDBOX_NETWORKING); + + build_dir.build(&self.toolchain, &krate, sandbox, |build| { + let res = self.execute_build(None, build)?; + if !res.successful { + bail!("failed to build dummy crate for {}", self.rustc_version); + } + + info!("copying essential files for {}", self.rustc_version); + let source = build.host_target_dir().join(&res.target).join("doc"); + let dest = ::tempdir::TempDir::new("essential-files")?; + + let files = ESSENTIAL_FILES_VERSIONED + .iter() + .map(|f| (f, true)) + .chain(ESSENTIAL_FILES_UNVERSIONED.iter().map(|f| (f, false))); + for (file, versioned) in files { + let segments = file.rsplitn(2, '.').collect::>(); + let file_name = if versioned { + format!("{}-{}.{}", segments[1], rustc_version, segments[0]) + } else { + file.to_string() + }; + let source_path = source.join(&file_name); + let dest_path = dest.path().join(&file_name); + ::std::fs::copy(&source_path, &dest_path).with_context(|_| { + format!( + "couldn't copy '{}' to '{}'", + source_path.display(), + dest_path.display() + ) + })?; + } + + let conn = connect_db()?; + add_path_into_database(&conn, "", &dest)?; + conn.query( + "INSERT INTO config (name, value) VALUES ('rustc_version', $1) \ + ON CONFLICT (name) DO UPDATE SET value = $1;", + &[&self.rustc_version.to_json()], + )?; + + Ok(()) + })?; + Ok(()) + } + + pub fn build_world(&mut self, doc_builder: &mut DocBuilder) -> Result<()> { + let mut count = 0; + crates_from_path( + &doc_builder.options().crates_io_index_path.clone(), + &mut |name, version| { + match self.build_package(doc_builder, name, version) { + Ok(status) => { + count += 1; + if status && count % 10 == 0 { + let _ = doc_builder.save_cache(); + } + } + Err(err) => warn!("failed to build package {} {}: {}", name, version, err), + } + doc_builder.add_to_cache(name, version); + }, + ) + } + + pub fn build_package( + &mut self, + doc_builder: &mut DocBuilder, + name: &str, + version: &str, + ) -> Result { + if !doc_builder.should_build(name, version) { + return Ok(false); + } + info!("building package {} {}", name, version); + + let conn = connect_db()?; + + let mut build_dir = self.workspace.build_dir(&format!("{}-{}", name, version)); + build_dir.purge()?; + + let krate = Crate::crates_io(name, version); + krate.fetch(&self.workspace)?; + + let sandbox = SandboxBuilder::new() + .memory_limit(Some(SANDBOX_MEMORY_LIMIT)) + .enable_networking(SANDBOX_NETWORKING); + + let res = build_dir.build(&self.toolchain, &krate, sandbox, |build| { + let mut files_list = None; + let mut has_docs = false; + let mut successful_targets = Vec::new(); + + // Do an initial build and then copy the sources in the database + let res = self.execute_build(None, &build)?; + if res.successful { + debug!("adding sources into database"); + let prefix = format!("sources/{}/{}", name, version); + files_list = Some(add_path_into_database( + &conn, + &prefix, + build.host_source_dir(), + )?); + + has_docs = build + .host_target_dir() + .join(&res.target) + .join("doc") + .join(name.replace("-", "_")) + .is_dir(); + } + + if has_docs { + debug!("adding documentation for the default target to the database"); + self.copy_docs( + &doc_builder, + &build.host_target_dir(), + name, + version, + &res.target, + true, + )?; + + // Then build the documentation for all the targets + for target in TARGETS { + debug!("building package {} {} for {}", name, version, target); + let target_res = self.execute_build(Some(target), &build)?; + if target_res.successful { + // Cargo is not giving any error and not generating documentation of some crates + // when we use a target compile options. Check documentation exists before + // adding target to successfully_targets. + if build.host_target_dir().join(target).join("doc").is_dir() { + debug!("adding documentation for target {} to the database", target); + self.copy_docs( + &doc_builder, + &build.host_target_dir(), + name, + version, + target, + false, + )?; + successful_targets.push(target.to_string()); + } + } + } + + self.upload_docs(doc_builder, &conn, name, version)?; + } + + let has_examples = build.host_source_dir().join("examples").is_dir(); + let release_id = add_package_into_database( + &conn, + res.cargo_metadata.root(), + &build.host_source_dir(), + &res, + files_list, + successful_targets, + has_docs, + has_examples, + )?; + add_build_into_database(&conn, &release_id, &res)?; + + doc_builder.add_to_cache(name, version); + Ok(res) + })?; + + build_dir.purge()?; + Ok(res.successful) + } + + fn execute_build(&self, target: Option<&str>, build: &Build) -> Result { + let metadata = Metadata::from_source_dir(&build.host_source_dir())?; + let cargo_metadata = + CargoMetadata::load(&self.workspace, &self.toolchain, &build.host_source_dir())?; + + let target = if let Some(target) = target { + target + } else if let Some(target) = metadata.default_target.as_ref().map(|s| s.as_str()) { + target + } else { + DEFAULT_TARGET + } + .to_string(); + + let mut rustdoc_flags: Vec = vec![ + "-Z".to_string(), + "unstable-options".to_string(), + "--resource-suffix".to_string(), + format!("-{}", parse_rustc_version(&self.rustc_version)?), + "--static-root-path".to_string(), + "/".to_string(), + "--disable-per-crate-search".to_string(), + ]; + for dep in &cargo_metadata.root_dependencies() { + rustdoc_flags.push("--extern-html-root-url".to_string()); + rustdoc_flags.push(format!( + "{}=https://docs.rs/{}/{}", + dep.name.replace("-", "_"), + dep.name, + dep.version + )); + } + let mut cargo_args = vec![ + "doc".to_owned(), + "--lib".to_owned(), + "--no-deps".to_owned(), + "--target".to_owned(), + target.to_owned(), + ]; + if let Some(features) = &metadata.features { + cargo_args.push("--features".to_owned()); + cargo_args.push(features.join(" ")); + } + if metadata.all_features { + cargo_args.push("--all-features".to_owned()); + } + if metadata.no_default_features { + cargo_args.push("--no-default-features".to_owned()); + } + + let mut storage = LogStorage::new(LevelFilter::Info); + storage.set_max_size(SANDBOX_MAX_LOG_SIZE); + storage.set_max_lines(SANDBOX_MAX_LOG_LINES); + + let successful = logging::capture(&storage, || { + build + .cargo() + .env( + "RUSTFLAGS", + metadata + .rustc_args + .map(|args| args.join("")) + .unwrap_or("".to_owned()), + ) + .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) + .args(&cargo_args) + .run() + .is_ok() + }); + + Ok(BuildResult { + build_log: storage.to_string(), + rustc_version: self.rustc_version.clone(), + docsrs_version: String::new(), // FIXME(pietro): fix + successful, + cargo_metadata, + target: target.to_string(), + }) + } + + fn copy_docs( + &self, + doc_builder: &DocBuilder, + target_dir: &Path, + name: &str, + version: &str, + target: &str, + is_default_target: bool, + ) -> Result<()> { + let source = target_dir.join(target); + + let mut dest = doc_builder.options().destination.join(name).join(version); + // only add target name to destination directory when we are copying a non-default target. + // this is allowing us to host documents in the root of the crate documentation directory. + // for example winapi will be available in docs.rs/winapi/$version/winapi/ for it's + // default target: x86_64-pc-windows-msvc. But since it will be built under + // cratesfyi/x86_64-pc-windows-msvc we still need target in this function. + if !is_default_target { + dest = dest.join(target); + } + + info!("{} {}", source.display(), dest.display()); + copy_doc_dir(source, dest, self.rustc_version.trim())?; + Ok(()) + } + + fn upload_docs( + &self, + doc_builder: &DocBuilder, + conn: &Connection, + name: &str, + version: &str, + ) -> Result<()> { + debug!("Adding documentation into database"); + let prefix = format!("rustdoc/{}/{}", name, version); + let database_prefix = + Path::new(&doc_builder.options().destination).join(format!("{}/{}", name, version)); + add_path_into_database(conn, &prefix, database_prefix)?; + Ok(()) + } +} + +pub(crate) struct BuildResult { + pub(crate) rustc_version: String, + pub(crate) docsrs_version: String, + pub(crate) build_log: String, + pub(crate) successful: bool, + target: String, + cargo_metadata: CargoMetadata, +} diff --git a/src/lib.rs b/src/lib.rs index c98890cd9..4ab8cdaf5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,9 +41,10 @@ extern crate futures; extern crate tokio; extern crate systemstat; extern crate rustwide; +extern crate tempdir; +pub use self::docbuilder::RustwideBuilder; pub use self::docbuilder::DocBuilder; -pub use self::docbuilder::ChrootBuilderResult; pub use self::docbuilder::options::DocBuilderOptions; pub use self::docbuilder::metadata::Metadata; pub use self::web::start_web_server; diff --git a/src/utils/build_doc.rs b/src/utils/build_doc.rs deleted file mode 100644 index be1f41d9b..000000000 --- a/src/utils/build_doc.rs +++ /dev/null @@ -1,245 +0,0 @@ -//! Crate documentation builder -//! -//! This module is extremely similar to cargo install operation, except it's building -//! documentation of a crate and not installing anything. - -use std::collections::HashSet; -use std::path::{Path, PathBuf}; -use std::env; -use std::sync::Arc; - -use cargo::core::{self, SourceId, Dependency, Source, Package, Workspace}; -use cargo::core::compiler::{DefaultExecutor, CompileMode, MessageFormat, BuildConfig, Executor}; -use cargo::core::package::PackageSet; -use cargo::core::registry::PackageRegistry; -use cargo::core::resolver; -use cargo::core::source::SourceMap; -use cargo::util::{CargoResult, Config, internal, Filesystem}; -use cargo::sources::SourceConfigMap; -use cargo::ops::{self, Packages, LibRule, FilterRule}; - -use utils::{get_current_versions, parse_rustc_version}; -use error::Result; - -use Metadata; - - -/// Builds documentation of a crate and version. -/// -/// Crate will be built into current working directory/crate-version. -/// -/// It will build latest version, if no version is given. -// idea is to make cargo to download -// and build a crate and its documentation -// instead of doing it manually like in the previous version of cratesfyi -pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> Result { - core::enable_nightly_features(); - let config = try!(Config::default()); - let source_id = try!(SourceId::crates_io(&config)); - - let source_cfg_map = try!(SourceConfigMap::new(&config)); - let mut source = try!(source_cfg_map.load(source_id, &HashSet::new())); - - let _lock = try!(config.acquire_package_cache_lock()); - - // update crates.io-index registry - try!(source.update()); - - let dep = try!(Dependency::parse_no_deprecated(name, vers, source_id)); - let deps = try!(source.query_vec(&dep)); - let pkgid = try!(deps.iter().map(|p| p.package_id()).max() - // FIXME: This is probably not a rusty way to handle options and results - // or maybe it is who knows... - .ok_or(internal("no package id available"))); - - let mut source_map = SourceMap::new(); - source_map.insert(source); - - let pkg_set = try!(PackageSet::new(&[pkgid.clone()], source_map, &config)); - - let pkg = try!(pkg_set.get_one(pkgid)).clone(); - - let current_dir = try!(env::current_dir()); - let target_dir = PathBuf::from(current_dir).join("cratesfyi"); - - let metadata = Metadata::from_package(&pkg).map_err(|e| internal(e.to_string()))?; - - // This is only way to pass rustc_args to cargo. - // CompileOptions::target_rustc_args is used only for the current crate, - // and since docs.rs never runs rustc on the current crate, we assume rustc_args - // will be used for the dependencies. That is why we are creating RUSTFLAGS environment - // variable instead of using target_rustc_args. - if let Some(rustc_args) = metadata.rustc_args { - env::set_var("RUSTFLAGS", rustc_args.join(" ")); - } - - // since https://github.com/rust-lang/rust/pull/48511 we can pass --resource-suffix to - // add correct version numbers to css and javascript files - let mut rustdoc_args: Vec = - vec!["-Z".to_string(), "unstable-options".to_string(), - "--resource-suffix".to_string(), - format!("-{}", parse_rustc_version(get_current_versions()?.0)?), - "--static-root-path".to_string(), "/".to_string(), - "--disable-per-crate-search".to_string()]; - - // since https://github.com/rust-lang/rust/pull/51384, we can pass --extern-html-root-url to - // force rustdoc to link to other docs.rs docs for dependencies - let source = try!(source_cfg_map.load(source_id, &HashSet::new())); - for (name, dep) in try!(resolve_deps(&pkg, &config, source)) { - rustdoc_args.push("--extern-html-root-url".to_string()); - rustdoc_args.push(format!("{}=https://docs.rs/{}/{}", - name.replace("-", "_"), dep.name(), dep.version())); - } - - if let Some(package_rustdoc_args) = metadata.rustdoc_args { - rustdoc_args.append(&mut package_rustdoc_args.iter().map(|s| s.to_owned()).collect()); - } - - let mut build_config = try!(BuildConfig::new(&config, - None, - &target.map(|t| t.to_string()), - CompileMode::Doc { deps: false })); - build_config.release = false; - build_config.message_format = MessageFormat::Human; - - let opts = ops::CompileOptions { - config: &config, - build_config, - features: metadata.features.unwrap_or(Vec::new()), - all_features: metadata.all_features, - no_default_features: metadata.no_default_features, - spec: Packages::Packages(Vec::new()), - filter: ops::CompileFilter::new(LibRule::True, - FilterRule::none(), - FilterRule::none(), - FilterRule::none(), - FilterRule::none()), - target_rustc_args: None, - target_rustdoc_args: Some(rustdoc_args), - local_rustdoc_args: None, - export_dir: None, - }; - - let ws = try!(Workspace::ephemeral(pkg, &config, Some(Filesystem::new(target_dir)), false)); - let exec: Arc = Arc::new(DefaultExecutor); - try!(ops::compile_ws(&ws, &opts, &exec)); - - Ok(try!(ws.current()).clone()) -} - -pub fn resolve_deps<'cfg>(pkg: &Package, config: &'cfg Config, src: Box) - -> CargoResult> -{ - let mut registry = try!(PackageRegistry::new(config)); - registry.add_preloaded(src); - registry.lock_patches(); - - let resolver = try!(resolver::resolve( - &[(pkg.summary().clone(), resolver::Method::Everything)], - pkg.manifest().replace(), - &mut registry, - &Default::default(), - None, - false, - )); - let dep_ids = resolver.deps(pkg.package_id()).map(|p| p.0).collect::>(); - let pkg_set = try!(registry.get(&dep_ids)); - let deps = try!(pkg_set.get_many(dep_ids)); - - let mut ret = Vec::new(); - for dep in deps { - if let Some(d) = pkg.dependencies().iter().find(|d| d.package_name() == dep.name()) { - ret.push((d.name_in_toml().to_string(), dep.clone())); - } - } - - Ok(ret) -} - -/// Downloads a crate and returns Cargo Package. -pub fn get_package(name: &str, vers: Option<&str>) -> CargoResult { - core::enable_nightly_features(); - debug!("Getting package with cargo"); - let config = try!(Config::default()); - let source_id = try!(SourceId::crates_io(&config)); - - let source_map = try!(SourceConfigMap::new(&config)); - let mut source = try!(source_map.load(source_id, &HashSet::new())); - - let _lock = try!(config.acquire_package_cache_lock()); - - try!(source.update()); - - let dep = try!(Dependency::parse_no_deprecated(name, vers, source_id)); - let deps = try!(source.query_vec(&dep)); - let pkgid = try!(deps.iter().map(|p| p.package_id()).max() - // FIXME: This is probably not a rusty way to handle options and results - // or maybe it is who knows... - .ok_or(internal("no package id available"))); - - let mut source_map = SourceMap::new(); - source_map.insert(source); - - let pkg_set = try!(PackageSet::new(&[pkgid.clone()], source_map, &config)); - - let pkg = try!(pkg_set.get_one(pkgid)).clone(); - - Ok(pkg) -} - - -/// Updates central crates-io.index repository -pub fn update_sources() -> CargoResult<()> { - let config = try!(Config::default()); - let source_id = try!(SourceId::crates_io(&config)); - - let _lock = try!(config.acquire_package_cache_lock()); - - let source_map = try!(SourceConfigMap::new(&config)); - let mut source = try!(source_map.load(source_id, &HashSet::new())); - - source.update() -} - - -/// Gets source path of a downloaded package. -pub fn source_path(pkg: &Package) -> Option<&Path> { - // parent of the manifest file is where source codes are stored - pkg.manifest_path().parent() -} - - - - -#[cfg(test)] -mod test { - use std::path::Path; - use super::*; - - #[test] - fn test_get_package() { - let pkg = get_package("rand", None); - assert!(pkg.is_ok()); - - let pkg = pkg.unwrap(); - - let manifest = pkg.manifest(); - assert_eq!(manifest.name().as_str(), "rand"); - } - - - #[test] - fn test_source_path() { - let pkg = get_package("rand", None).unwrap(); - let source_path = source_path(&pkg).unwrap(); - assert!(source_path.is_dir()); - - let cargo_toml_path = Path::new(source_path).join("Cargo.toml"); - assert!(cargo_toml_path.exists()); - assert!(cargo_toml_path.is_file()); - - let src_path = Path::new(source_path).join("src"); - assert!(src_path.exists()); - assert!(src_path.is_dir()); - } -} diff --git a/src/utils/build_doc_rustwide.rs b/src/utils/build_doc_rustwide.rs deleted file mode 100644 index a39e8da47..000000000 --- a/src/utils/build_doc_rustwide.rs +++ /dev/null @@ -1,146 +0,0 @@ -use error::Result; -use log::LevelFilter; -use rustwide::{ - cmd::{Command, SandboxBuilder}, - logging::{self, LogStorage}, - Crate, Toolchain, Workspace, WorkspaceBuilder, -}; -use std::path::Path; -use utils::cargo_metadata::CargoMetadata; -use utils::parse_rustc_version; -use Metadata; - -// TODO: 1GB might not be enough -const SANDBOX_MEMORY_LIMIT: usize = 1024 * 1024 * 1024; // 1GB -const SANDBOX_NETWORKING: bool = false; -const SANDBOX_MAX_LOG_SIZE: usize = 1024 * 1024; // 1MB -const SANDBOX_MAX_LOG_LINES: usize = 10_000; - -pub fn build_doc_rustwide( - name: &str, - version: &str, - target: Option<&str>, -) -> Result { - // TODO: Handle workspace path correctly - let workspace = WorkspaceBuilder::new(Path::new("/tmp/docs-builder"), "docsrs").init()?; - - // TODO: Instead of using just nightly, we can pin a version. - // Docs.rs can only use nightly (due to unstable docs.rs features in rustdoc) - let toolchain = Toolchain::Dist { - name: "nightly".into(), - }; - toolchain.install(&workspace)?; - if let Some(target) = target { - toolchain.add_target(&workspace, target)?; - } - - let krate = Crate::crates_io(name, version); - krate.fetch(&workspace)?; - - let sandbox = SandboxBuilder::new() - .memory_limit(Some(SANDBOX_MEMORY_LIMIT)) - .enable_networking(SANDBOX_NETWORKING); - - let mut build_dir = workspace.build_dir(&format!("{}-{}", name, version)); - let pkg = build_dir.build(&toolchain, &krate, sandbox, |build| { - let metadata = Metadata::from_source_dir(&build.host_source_dir())?; - let cargo_metadata = CargoMetadata::load(&workspace, &toolchain, &build.host_source_dir())?; - - let mut rustdoc_flags: Vec = vec![ - "-Z".to_string(), - "unstable-options".to_string(), - "--resource-suffix".to_string(), - format!( - "-{}", - parse_rustc_version(rustc_version(&workspace, &toolchain)?)? - ), - "--static-root-path".to_string(), - "/".to_string(), - "--disable-per-crate-search".to_string(), - ]; - - for dep in &cargo_metadata.root_dependencies() { - rustdoc_flags.push("--extern-html-root-url".to_string()); - rustdoc_flags.push(format!( - "{}=https://docs.rs/{}/{}", - dep.name.replace("-", "_"), - dep.name, - dep.version - )); - } - - let mut cargo_args = vec!["doc".to_owned(), "--lib".to_owned(), "--no-deps".to_owned()]; - if let Some(features) = &metadata.features { - cargo_args.push("--features".to_owned()); - cargo_args.push(features.join(" ")); - } - if metadata.all_features { - cargo_args.push("--all-features".to_owned()); - } - if metadata.no_default_features { - cargo_args.push("--no-default-features".to_owned()); - } - if let Some(target) = target { - cargo_args.push("--target".into()); - cargo_args.push(target.into()); - } - - let mut storage = LogStorage::new(LevelFilter::Info); - storage.set_max_size(SANDBOX_MAX_LOG_SIZE); - storage.set_max_lines(SANDBOX_MAX_LOG_LINES); - - logging::capture(&storage, || { - build - .cargo() - .env( - "RUSTFLAGS", - metadata - .rustc_args - .map(|args| args.join("")) - .unwrap_or("".to_owned()), - ) - .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) - .args(&cargo_args) - .run() - })?; - - // TODO: We need to return build result as well - Ok(BuildDocOutput { - package_version: cargo_metadata.root().version.to_string(), - build_log: storage.to_string(), - }) - })?; - - Ok(pkg) -} - -pub struct BuildDocOutput { - package_version: String, - build_log: String, -} - -impl BuildDocOutput { - pub fn package_version(&self) -> &str { - &self.package_version - } - - pub fn build_log(&self) -> &str { - &self.build_log - } -} - -fn rustc_version(workspace: &Workspace, toolchain: &Toolchain) -> Result { - let res = Command::new(workspace, toolchain.rustc()) - .args(&["--version"]) - .log_output(false) - .run_capture()?; - - let mut iter = res.stdout_lines().iter(); - if let (Some(line), None) = (iter.next(), iter.next()) { - Ok(line.clone()) - } else { - Err(::failure::err_msg( - "invalid output returned by `rustc --version`", - )) - } -} diff --git a/src/utils/cargo_metadata.rs b/src/utils/cargo_metadata.rs index a1a22faab..79f09bb48 100644 --- a/src/utils/cargo_metadata.rs +++ b/src/utils/cargo_metadata.rs @@ -1,8 +1,7 @@ -use cargo::core::Package as CargoLibPackage; use error::Result; use rustwide::{cmd::Command, Toolchain, Workspace}; use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; +use std::path::Path; pub(crate) struct CargoMetadata { packages: HashMap, @@ -48,61 +47,6 @@ impl CargoMetadata { }) } - // All of this is very hacky, but it's just needed to cleanly transition the code from cargo - // lib to cargo metadata. - pub(crate) fn from_cargo_lib_package(pkg: &CargoLibPackage) -> Self { - let id = pkg.package_id().to_string(); - CargoMetadata { - packages: ::std::iter::once(( - id.clone(), - Package { - id: id.clone(), - name: pkg.name().as_str().to_string(), - version: pkg.version().to_string(), - license: pkg.manifest().metadata().license.clone(), - repository: pkg.manifest().metadata().repository.clone(), - homepage: pkg.manifest().metadata().homepage.clone(), - description: pkg.manifest().metadata().description.clone(), - documentation: pkg.manifest().metadata().documentation.clone(), - targets: pkg - .manifest() - .targets() - .iter() - .map(|target| Target { - name: target.name().to_string(), - kind: if target.is_lib() { - vec!["lib".into()] - } else { - vec![] - }, - src_path: target.src_path().path().map(|p| p.into()), - }) - .collect(), - dependencies: pkg - .manifest() - .dependencies() - .iter() - .map(|dep| Dependency { - name: dep.package_name().to_string(), - req: dep.version_req().to_string(), - kind: match dep.kind() { - ::cargo::core::dependency::Kind::Normal => None, - ::cargo::core::dependency::Kind::Development => Some("dev".into()), - ::cargo::core::dependency::Kind::Build => Some("build".into()), - }, - }) - .collect(), - readme: pkg.manifest().metadata().readme.clone(), - keywords: pkg.manifest().metadata().keywords.clone(), - authors: pkg.manifest().metadata().authors.clone(), - }, - )) - .collect(), - deps_graph: HashMap::new(), - root_id: id, - } - } - pub(crate) fn root_dependencies(&self) -> Vec<&Package> { let ids = &self.deps_graph[&self.root_id]; self.packages @@ -138,7 +82,7 @@ pub(crate) struct Package { pub(crate) struct Target { pub(crate) name: String, pub(crate) kind: Vec, - pub(crate) src_path: Option, + pub(crate) src_path: Option, } #[derive(RustcDecodable)] diff --git a/src/utils/copy.rs b/src/utils/copy.rs index bc73c8dd1..1d11057a5 100644 --- a/src/utils/copy.rs +++ b/src/utils/copy.rs @@ -84,7 +84,6 @@ fn copy_files_and_handle_html(source: PathBuf, #[cfg(test)] mod test { extern crate env_logger; - extern crate tempdir; use std::fs; use std::path::Path; use super::*; diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index d8c40677c..030c1a8f9 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,12 +9,13 @@ use std::process::exit; use std::fs::File; use std::io::Write; use std::time::Duration; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use libc::fork; use time; +use docbuilder::RustwideBuilder; use DocBuilderOptions; use DocBuilder; -use utils::{update_sources, update_release_activity, github_updater, pubsubhubbub}; +use utils::{update_release_activity, github_updater, pubsubhubbub}; use db::{connect_db, update_search_index}; @@ -85,6 +86,8 @@ pub fn start_daemon() { QueueInProgress(usize), } + let mut builder = RustwideBuilder::init(&Path::new("rustwide")).unwrap(); + let mut status = BuilderState::Fresh; loop { @@ -116,11 +119,6 @@ pub fn start_daemon() { if let Err(e) = doc_builder.save_cache() { error!("Failed to save cache: {}", e); } - - if let Err(e) = update_sources() { - error!("Failed to update sources: {}", e); - continue; - } } // Only build crates if there are any to build @@ -158,18 +156,13 @@ pub fn start_daemon() { error!("Failed to load cache: {}", e); continue; } - - if let Err(e) = update_sources() { - error!("Failed to update sources: {}", e); - continue; - } } // Run build_packages_queue under `catch_unwind` to catch panics // This only panicked twice in the last 6 months but its just a better // idea to do this. let res = catch_unwind(AssertUnwindSafe(|| { - match doc_builder.build_next_queue_package() { + match doc_builder.build_next_queue_package(&mut builder) { Err(e) => error!("Failed to build crate from queue: {}", e), Ok(crate_built) => if crate_built { status.increment(); diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 96a81aa6c..8db86bd13 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,8 +1,6 @@ //! Various utilities for cratesfyi -pub use self::build_doc::{build_doc, get_package, source_path, update_sources, resolve_deps}; -pub use self::build_doc_rustwide::build_doc_rustwide; pub use self::copy::{copy_dir, copy_doc_dir}; pub use self::github_updater::github_updater; pub use self::release_activity_updater::update_release_activity; @@ -14,8 +12,6 @@ pub(crate) use self::cargo_metadata::{CargoMetadata, Package as MetadataPackage} mod cargo_metadata; mod github_updater; -mod build_doc; -mod build_doc_rustwide; mod copy; mod release_activity_updater; mod daemon; From dcceaff9f2049dbdea51ab458763c36d3d3c3429 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 13 Sep 2019 11:50:50 +0200 Subject: [PATCH 12/23] record the docsrs commit in rustwide builds --- src/docbuilder/rustwide_builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 59e088412..ecdbc8d8b 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -380,7 +380,7 @@ impl RustwideBuilder { Ok(BuildResult { build_log: storage.to_string(), rustc_version: self.rustc_version.clone(), - docsrs_version: String::new(), // FIXME(pietro): fix + docsrs_version: format!("docsrs {}", ::BUILD_VERSION), successful, cargo_metadata, target: target.to_string(), From aa174e4450b2882e299f5e2e2441ebbbb90113d1 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 13 Sep 2019 11:55:09 +0200 Subject: [PATCH 13/23] purge more rustwide stuff when it's not needed anymore --- src/docbuilder/rustwide_builder.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index ecdbc8d8b..37f4e4241 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -73,6 +73,7 @@ impl RustwideBuilder { .command_timeout(COMMAND_TIMEOUT) .command_no_output_timeout(COMMAND_NO_OUTPUT_TIMEOUT) .init()?; + workspace.purge_all_build_dirs()?; let toolchain = Toolchain::Dist { name: "nightly".into(), @@ -180,6 +181,9 @@ impl RustwideBuilder { Ok(()) })?; + + build_dir.purge()?; + krate.purge_from_cache(&self.workspace)?; Ok(()) } @@ -304,6 +308,7 @@ impl RustwideBuilder { })?; build_dir.purge()?; + krate.purge_from_cache(&self.workspace)?; Ok(res.successful) } From 7137705c46eaec3e981871c7a17f1b901fc60946 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 13 Sep 2019 12:09:51 +0200 Subject: [PATCH 14/23] remove ignored and broken test --- src/utils/copy.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/utils/copy.rs b/src/utils/copy.rs index 1d11057a5..28847dbc1 100644 --- a/src/utils/copy.rs +++ b/src/utils/copy.rs @@ -100,24 +100,4 @@ mod test { assert!(res.is_ok()); } - - - #[test] - #[ignore] - fn test_copy_doc_dir() { - // lets build documentation of rand crate - use utils::build_doc; - let pkg = build_doc("rand", None, None).unwrap(); - - let pkg_dir = format!("rand-{}", pkg.manifest().version()); - let target = Path::new(&pkg_dir); - let destination = tempdir::TempDir::new("cratesfyi").unwrap(); - let res = copy_doc_dir(target, destination.path(), "UNKNOWN"); - - // remove build and temp dir - fs::remove_dir_all(target).unwrap(); - fs::remove_dir_all(destination.path()).unwrap(); - - assert!(res.is_ok()); - } } From 77372e2b4324670fd9d6aa7c90d5709904b9cb8d Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 13 Sep 2019 12:43:19 +0200 Subject: [PATCH 15/23] remove outdated flags and environment variables --- src/bin/cratesfyi.rs | 27 --------------------------- src/docbuilder/options.rs | 27 +++++---------------------- 2 files changed, 5 insertions(+), 49 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 34a28a47c..1c38e2788 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -35,21 +35,6 @@ pub fn main() { .long("destination") .help("Sets destination path") .takes_value(true)) - .arg(Arg::with_name("CHROOT_PATH") - .short("c") - .long("chroot-path") - .help("Sets chroot path") - .takes_value(true)) - .arg(Arg::with_name("CHROOT_USER") - .short("u") - .long("chroot-user") - .help("Sets chroot user name") - .takes_value(true)) - .arg(Arg::with_name("CONTAINER_NAME") - .short("n") - .long("container-name") - .help("Sets name of the container") - .takes_value(true)) .arg(Arg::with_name("CRATES_IO_INDEX_PATH") .long("crates-io-index-path") .help("Sets crates.io-index path") @@ -154,18 +139,6 @@ pub fn main() { docbuilder_opts.destination = PathBuf::from(destination); } - if let Some(chroot_path) = matches.value_of("CHROOT_PATH") { - docbuilder_opts.chroot_path = PathBuf::from(chroot_path); - } - - if let Some(chroot_user) = matches.value_of("CHROOT_USER") { - docbuilder_opts.chroot_user = chroot_user.to_string(); - } - - if let Some(container_name) = matches.value_of("CONTAINER_NAME") { - docbuilder_opts.container_name = container_name.to_string(); - } - if let Some(crates_io_index_path) = matches.value_of("CRATES_IO_INDEX_PATH") { docbuilder_opts.crates_io_index_path = PathBuf::from(crates_io_index_path); } diff --git a/src/docbuilder/options.rs b/src/docbuilder/options.rs index 4726acc63..28d5c63b4 100644 --- a/src/docbuilder/options.rs +++ b/src/docbuilder/options.rs @@ -9,9 +9,6 @@ pub struct DocBuilderOptions { pub keep_build_directory: bool, pub prefix: PathBuf, pub destination: PathBuf, - pub chroot_path: PathBuf, - pub chroot_user: String, - pub container_name: String, pub crates_io_index_path: PathBuf, pub skip_if_exists: bool, pub skip_if_log_exists: bool, @@ -27,18 +24,14 @@ impl Default for DocBuilderOptions { let cwd = env::current_dir().unwrap(); - let (prefix, destination, chroot_path, crates_io_index_path) = + let (prefix, destination, crates_io_index_path) = generate_paths(cwd); DocBuilderOptions { prefix: prefix, destination: destination, - chroot_path: chroot_path, crates_io_index_path: crates_io_index_path, - chroot_user: "cratesfyi".to_string(), - container_name: env::var("CRATESFYI_CONTAINER_NAME").expect("CRATESFYI_CONTAINER_NAME"), - keep_build_directory: false, skip_if_exists: false, skip_if_log_exists: false, @@ -53,16 +46,12 @@ impl Default for DocBuilderOptions { impl fmt::Debug for DocBuilderOptions { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, - "DocBuilderOptions {{ destination: {:?}, chroot_path: {:?}, \ + "DocBuilderOptions {{ destination: {:?}, \ crates_io_index_path: {:?}, \ - container_name: {:?}, chroot_user: {:?}, \ keep_build_directory: {:?}, skip_if_exists: {:?}, \ skip_if_log_exists: {:?}, debug: {:?} }}", self.destination, - self.chroot_path, self.crates_io_index_path, - self.container_name, - self.chroot_user, self.keep_build_directory, self.skip_if_exists, self.skip_if_log_exists, @@ -74,12 +63,11 @@ impl fmt::Debug for DocBuilderOptions { impl DocBuilderOptions { /// Creates new DocBuilderOptions from prefix pub fn from_prefix(prefix: PathBuf) -> DocBuilderOptions { - let (prefix, destination, chroot_path, crates_io_index_path) = + let (prefix, destination, crates_io_index_path) = generate_paths(prefix); DocBuilderOptions { prefix: prefix, destination: destination, - chroot_path: chroot_path, crates_io_index_path: crates_io_index_path, ..Default::default() @@ -91,9 +79,6 @@ impl DocBuilderOptions { if !self.destination.exists() { bail!("destination path '{}' does not exist", self.destination.display()); } - if !self.chroot_path.exists() { - bail!("chroot path '{}' does not exist", self.chroot_path.display()); - } if !self.crates_io_index_path.exists() { bail!("crates.io-index path '{}' does not exist", self.crates_io_index_path.display()); } @@ -103,11 +88,9 @@ impl DocBuilderOptions { -fn generate_paths(prefix: PathBuf) -> (PathBuf, PathBuf, PathBuf, PathBuf) { - let name = env::var_os("CRATESFYI_CONTAINER_NAME").expect("CRATESFYI_CONTAINER_NAME"); +fn generate_paths(prefix: PathBuf) -> (PathBuf, PathBuf, PathBuf) { let destination = PathBuf::from(&prefix).join("documentations"); - let chroot_path = PathBuf::from(&prefix).join(&name).join("rootfs"); let crates_io_index_path = PathBuf::from(&prefix).join("crates.io-index"); - (prefix, destination, chroot_path, crates_io_index_path) + (prefix, destination, crates_io_index_path) } From 336fd5d5a0ac35885bf549040655a4ba331b9b71 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 13 Sep 2019 13:28:41 +0200 Subject: [PATCH 16/23] allow limits to be overridden and show them in the ui --- src/db/migrate.rs | 14 +++++ src/docbuilder/limits.rs | 97 ++++++++++++++++++++++++++++++ src/docbuilder/mod.rs | 2 + src/docbuilder/rustwide_builder.rs | 47 +++++++-------- src/web/builds.rs | 5 ++ src/web/sitemap.rs | 6 +- templates/about.hbs | 24 ++++++++ templates/builds.hbs | 31 ++++++++-- 8 files changed, 194 insertions(+), 32 deletions(-) create mode 100644 src/docbuilder/limits.rs diff --git a/src/db/migrate.rs b/src/db/migrate.rs index c71a65738..c1adc1f12 100644 --- a/src/db/migrate.rs +++ b/src/db/migrate.rs @@ -179,6 +179,20 @@ pub fn migrate(version: Option) -> CratesfyiResult<()> { // downgrade query "ALTER TABLE queue DROP COLUMN priority;" ), + migration!( + // version + 3, + // description + "Added sandbox_overrides table", + // upgrade query + "CREATE TABLE sandbox_overrides ( + crate_name VARCHAR NOT NULL PRIMARY KEY, + max_memory_bytes INTEGER, + timeout_seconds INTEGER + );", + // downgrade query + "DROP TABLE sandbox_overrides;" + ), ]; for migration in migrations { diff --git a/src/docbuilder/limits.rs b/src/docbuilder/limits.rs new file mode 100644 index 000000000..9ba9d2b5b --- /dev/null +++ b/src/docbuilder/limits.rs @@ -0,0 +1,97 @@ +use error::Result; +use postgres::Connection; +use std::collections::BTreeMap; +use std::time::Duration; + +const DEFAULT_MEMORY_LIMIT: usize = 3 * 1024 * 1024 * 1024; // 3 GB +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(15 * 60); // 15 minutes +const DEFAULT_NETWORKING: bool = false; +const DEFAULT_MAX_LOG_SIZE: usize = 100 * 1024; // 100 KB + +pub(crate) struct Limits { + memory: usize, + timeout: Duration, + networking: bool, + max_log_size: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + memory: DEFAULT_MEMORY_LIMIT, + timeout: DEFAULT_TIMEOUT, + networking: DEFAULT_NETWORKING, + max_log_size: DEFAULT_MAX_LOG_SIZE, + } + } +} + +impl Limits { + pub(crate) fn for_crate(conn: &Connection, name: &str) -> Result { + let mut limits = Self::default(); + + let res = conn.query( + "SELECT * FROM sandbox_overrides WHERE crate_name = $1;", + &[&name], + )?; + if !res.is_empty() { + let row = res.get(0); + if let Some(memory) = row.get::<_, Option>("max_memory_bytes") { + limits.memory = memory as usize; + } + if let Some(timeout) = row.get::<_, Option>("timeout_seconds") { + limits.timeout = Duration::from_secs(timeout as u64); + } + } + + Ok(limits) + } + + pub(crate) fn memory(&self) -> usize { + self.memory + } + + pub(crate) fn timeout(&self) -> Duration { + self.timeout + } + + pub(crate) fn networking(&self) -> bool { + self.networking + } + + pub(crate) fn max_log_size(&self) -> usize { + self.max_log_size + } + + pub(crate) fn for_website(&self) -> BTreeMap { + let time_scale = |v| scale(v, 60, &["seconds", "minutes", "hours"]); + let size_scale = |v| scale(v, 1024, &["bytes", "KB", "MB", "GB"]); + + let mut res = BTreeMap::new(); + res.insert("Available RAM".into(), size_scale(self.memory)); + res.insert( + "Maximum rustdoc execution time".into(), + time_scale(self.timeout.as_secs() as usize), + ); + res.insert("Maximum size of a build log".into(), size_scale(self.max_log_size)); + if self.networking { + res.insert("Network access".into(), "allowed".into()); + } else { + res.insert("Network access".into(), "blocked".into()); + } + res + } +} + +fn scale(mut value: usize, interval: usize, labels: &[&str]) -> String { + let mut chosen_label = &labels[0]; + for label in &labels[1..] { + if (value as f64) / (interval as f64) >= 1.0 { + chosen_label = label; + value = value / interval; + } else { + break; + } + } + format!("{} {}", value, chosen_label) +} diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index d4bdca33d..9087ffe14 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -1,12 +1,14 @@ pub mod options; pub mod metadata; +mod limits; mod rustwide_builder; mod crates; mod queue; pub use self::rustwide_builder::RustwideBuilder; pub(crate) use self::rustwide_builder::BuildResult; +pub(crate) use self::limits::Limits; use std::fs; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 37f4e4241..540016d6b 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1,7 +1,7 @@ use super::DocBuilder; use db::file::add_path_into_database; use db::{add_build_into_database, add_package_into_database, connect_db}; -use docbuilder::crates::crates_from_path; +use docbuilder::{crates::crates_from_path, Limits}; use error::Result; use failure::ResultExt; use log::LevelFilter; @@ -11,18 +11,9 @@ use rustwide::cmd::{Command, SandboxBuilder}; use rustwide::logging::{self, LogStorage}; use rustwide::{Build, Crate, Toolchain, Workspace, WorkspaceBuilder}; use std::path::Path; -use std::time::Duration; use utils::{copy_doc_dir, parse_rustc_version, CargoMetadata}; use Metadata; -// TODO: 1GB might not be enough -const SANDBOX_MEMORY_LIMIT: usize = 1024 * 1024 * 1024; // 1GB -const SANDBOX_NETWORKING: bool = false; -const SANDBOX_MAX_LOG_SIZE: usize = 1024 * 1024; // 1MB -const SANDBOX_MAX_LOG_LINES: usize = 10_000; -const COMMAND_TIMEOUT: Option = Some(Duration::from_secs(60 * 60)); // 1 hour -const COMMAND_NO_OUTPUT_TIMEOUT: Option = None; - static USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; static TARGETS: &[&str] = &[ "i686-apple-darwin", @@ -61,6 +52,9 @@ static ESSENTIAL_FILES_UNVERSIONED: &[&str] = &[ "SourceSerifPro-It.ttf.woff", ]; +static DUMMY_CRATE_NAME: &str = "acme-client"; +static DUMMY_CRATE_VERSION: &str = "0.0.0"; + pub struct RustwideBuilder { workspace: Workspace, toolchain: Toolchain, @@ -69,10 +63,7 @@ pub struct RustwideBuilder { impl RustwideBuilder { pub fn init(workspace_path: &Path) -> Result { - let workspace = WorkspaceBuilder::new(workspace_path, USER_AGENT) - .command_timeout(COMMAND_TIMEOUT) - .command_no_output_timeout(COMMAND_NO_OUTPUT_TIMEOUT) - .init()?; + let workspace = WorkspaceBuilder::new(workspace_path, USER_AGENT).init()?; workspace.purge_all_build_dirs()?; let toolchain = Toolchain::Dist { @@ -126,21 +117,24 @@ impl RustwideBuilder { info!("building a dummy crate to get essential files"); let rustc_version = parse_rustc_version(&self.rustc_version)?; + let conn = connect_db()?; + let limits = Limits::for_crate(&conn, DUMMY_CRATE_NAME)?; + let mut build_dir = self .workspace .build_dir(&format!("essential-files-{}", rustc_version)); build_dir.purge()?; // acme-client-0.0.0 is an empty library crate and it will always build - let krate = Crate::crates_io("acme-client", "0.0.0"); + let krate = Crate::crates_io(DUMMY_CRATE_NAME, DUMMY_CRATE_VERSION); krate.fetch(&self.workspace)?; let sandbox = SandboxBuilder::new() - .memory_limit(Some(SANDBOX_MEMORY_LIMIT)) - .enable_networking(SANDBOX_NETWORKING); + .memory_limit(Some(limits.memory())) + .enable_networking(limits.networking()); build_dir.build(&self.toolchain, &krate, sandbox, |build| { - let res = self.execute_build(None, build)?; + let res = self.execute_build(None, build, &limits)?; if !res.successful { bail!("failed to build dummy crate for {}", self.rustc_version); } @@ -171,7 +165,6 @@ impl RustwideBuilder { })?; } - let conn = connect_db()?; add_path_into_database(&conn, "", &dest)?; conn.query( "INSERT INTO config (name, value) VALUES ('rustc_version', $1) \ @@ -218,6 +211,7 @@ impl RustwideBuilder { info!("building package {} {}", name, version); let conn = connect_db()?; + let limits = Limits::for_crate(&conn, name)?; let mut build_dir = self.workspace.build_dir(&format!("{}-{}", name, version)); build_dir.purge()?; @@ -226,8 +220,8 @@ impl RustwideBuilder { krate.fetch(&self.workspace)?; let sandbox = SandboxBuilder::new() - .memory_limit(Some(SANDBOX_MEMORY_LIMIT)) - .enable_networking(SANDBOX_NETWORKING); + .memory_limit(Some(limits.memory())) + .enable_networking(limits.networking()); let res = build_dir.build(&self.toolchain, &krate, sandbox, |build| { let mut files_list = None; @@ -235,7 +229,7 @@ impl RustwideBuilder { let mut successful_targets = Vec::new(); // Do an initial build and then copy the sources in the database - let res = self.execute_build(None, &build)?; + let res = self.execute_build(None, &build, &limits)?; if res.successful { debug!("adding sources into database"); let prefix = format!("sources/{}/{}", name, version); @@ -267,7 +261,7 @@ impl RustwideBuilder { // Then build the documentation for all the targets for target in TARGETS { debug!("building package {} {} for {}", name, version, target); - let target_res = self.execute_build(Some(target), &build)?; + let target_res = self.execute_build(Some(target), &build, &limits)?; if target_res.successful { // Cargo is not giving any error and not generating documentation of some crates // when we use a target compile options. Check documentation exists before @@ -312,7 +306,7 @@ impl RustwideBuilder { Ok(res.successful) } - fn execute_build(&self, target: Option<&str>, build: &Build) -> Result { + fn execute_build(&self, target: Option<&str>, build: &Build, limits: &Limits) -> Result { let metadata = Metadata::from_source_dir(&build.host_source_dir())?; let cargo_metadata = CargoMetadata::load(&self.workspace, &self.toolchain, &build.host_source_dir())?; @@ -363,12 +357,13 @@ impl RustwideBuilder { } let mut storage = LogStorage::new(LevelFilter::Info); - storage.set_max_size(SANDBOX_MAX_LOG_SIZE); - storage.set_max_lines(SANDBOX_MAX_LOG_LINES); + storage.set_max_size(limits.max_log_size()); let successful = logging::capture(&storage, || { build .cargo() + .timeout(Some(limits.timeout())) + .no_output_timeout(None) .env( "RUSTFLAGS", metadata diff --git a/src/web/builds.rs b/src/web/builds.rs index a0eb05530..a8fed29ed 100644 --- a/src/web/builds.rs +++ b/src/web/builds.rs @@ -1,5 +1,6 @@ +use docbuilder::Limits; use std::collections::BTreeMap; use super::MetaData; use super::pool::Pool; @@ -27,6 +28,7 @@ struct BuildsPage { metadata: Option, builds: Vec, build_details: Option, + limits: Limits, } @@ -54,6 +56,7 @@ impl ToJson for BuildsPage { m.insert("metadata".to_owned(), self.metadata.to_json()); m.insert("builds".to_owned(), self.builds.to_json()); m.insert("build_details".to_owned(), self.build_details.to_json()); + m.insert("limits".into(), self.limits.for_website().to_json()); m.to_json() } } @@ -67,6 +70,7 @@ pub fn build_list_handler(req: &mut Request) -> IronResult { let req_build_id: i32 = router.find("id").unwrap_or("0").parse().unwrap_or(0); let conn = extension!(req, Pool); + let limits = ctry!(Limits::for_crate(&conn, name)); let mut build_list: Vec = Vec::new(); let mut build_details = None; @@ -131,6 +135,7 @@ pub fn build_list_handler(req: &mut Request) -> IronResult { metadata: MetaData::from_crate(&conn, &name, &version), builds: build_list, build_details: build_details, + limits, }; Page::new(builds_page) .set_true("show_package_navigation") diff --git a/src/web/sitemap.rs b/src/web/sitemap.rs index 4bab4fadc..5c8b93ae5 100644 --- a/src/web/sitemap.rs +++ b/src/web/sitemap.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use iron::prelude::*; use iron::headers::ContentType; -use rustc_serialize::json::Json; +use rustc_serialize::json::{Json, ToJson}; use super::page::Page; use super::pool::Pool; use time; @@ -40,10 +40,12 @@ pub fn about_handler(req: &mut Request) -> IronResult { if let Some(row) = res.iter().next() { if let Some(Ok::(res)) = row.get_opt(0) { if let Some(vers) = res.as_string() { - content.insert("rustc_version".to_string(), vers.to_string()); + content.insert("rustc_version".to_string(), vers.to_json()); } } } + content.insert("limits".to_string(), ::docbuilder::Limits::default().for_website().to_json()); + Page::new(content).title("About Docs.rs").to_resp("about") } diff --git a/templates/about.hbs b/templates/about.hbs index f602ff403..470ea67f2 100644 --- a/templates/about.hbs +++ b/templates/about.hbs @@ -31,6 +31,30 @@ no README will be displayed.

+

Global sandbox limits

+ +

+ All the builds on docs.rs are executed inside a sandbox with limited + resources. The current limits are the following: +

+ + + + {{#each content.limits}} + + + + + {{/each}} + +
{{{@key}}}{{this}}
+ +

+ If a build fails because it hit one of those limits please + open an issue + to get them increased for your crate. +

+

Redirections

diff --git a/templates/builds.hbs b/templates/builds.hbs index c32fac506..630661ea7 100644 --- a/templates/builds.hbs +++ b/templates/builds.hbs @@ -8,14 +8,12 @@

Build #{{build_details.id}} {{build_details.build_time}}
- 
-$ rustc --version
+    
$ rustc --version
 {{build_details.rustc_version}}
 $ cratesfyi --version
 {{build_details.cratesfyi_version}}
 $ cratesfyi ...
-{{build_details.output}}
-    

+{{build_details.output}}
{{/if}}
@@ -36,6 +34,31 @@ $ cratesfyi ... {{/each}} + +
+

{{metadata.name}}'s sandbox limits

+ +

+ All the builds on docs.rs are executed inside a sandbox with limited + resources. The limits for this crate are the following: +

+ + + + {{#each limits}} + + + + + {{/each}} + +
{{{@key}}}{{this}}
+

+ If a build fails because it hit one of those limits please + open an issue + to get them increased. +

+
{{/with}} From 6e7ab9f9f970c63c238aa645701eed31dd8843a5 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 13 Sep 2019 15:46:14 +0200 Subject: [PATCH 17/23] make the rustwide workspace configurable via envvar --- .gitignore | 2 +- src/bin/cratesfyi.rs | 8 ++++---- src/docbuilder/rustwide_builder.rs | 18 +++++++++++++++--- src/utils/daemon.rs | 4 ++-- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 23a5c4c78..e5e7c1e51 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ target *.css.map .sass-cache .vagrant -/rustwide +.rustwide diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 1c38e2788..b048c9b82 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -9,7 +9,7 @@ extern crate rustwide; use std::env; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use clap::{Arg, App, SubCommand}; use cratesfyi::{DocBuilder, RustwideBuilder, DocBuilderOptions, db}; @@ -156,18 +156,18 @@ pub fn main() { if let Some(_) = matches.subcommand_matches("world") { docbuilder.load_cache().expect("Failed to load cache"); - let mut builder = RustwideBuilder::init(&Path::new("rustwide")).unwrap(); + let mut builder = RustwideBuilder::init().unwrap(); builder.build_world(&mut docbuilder).expect("Failed to build world"); docbuilder.save_cache().expect("Failed to save cache"); } else if let Some(matches) = matches.subcommand_matches("crate") { docbuilder.load_cache().expect("Failed to load cache"); - let mut builder = RustwideBuilder::init(&Path::new("rustwide")).unwrap(); + let mut builder = RustwideBuilder::init().unwrap(); builder.build_package(&mut docbuilder, matches.value_of("CRATE_NAME").unwrap(), matches.value_of("CRATE_VERSION").unwrap()) .expect("Building documentation failed"); docbuilder.save_cache().expect("Failed to save cache"); } else if let Some(_) = matches.subcommand_matches("add-essential-files") { - let builder = RustwideBuilder::init(&Path::new("rustwide")).unwrap(); + let builder = RustwideBuilder::init().unwrap(); builder.add_essential_files().expect("failed to add essential files"); } else if let Some(_) = matches.subcommand_matches("lock") { docbuilder.lock().expect("Failed to lock"); diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 540016d6b..77843568b 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -15,6 +15,8 @@ use utils::{copy_doc_dir, parse_rustc_version, CargoMetadata}; use Metadata; static USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; +static DEFAULT_RUSTWIDE_WORKSPACE: &str = ".rustwide"; + static TARGETS: &[&str] = &[ "i686-apple-darwin", "i686-pc-windows-msvc", @@ -62,8 +64,13 @@ pub struct RustwideBuilder { } impl RustwideBuilder { - pub fn init(workspace_path: &Path) -> Result { - let workspace = WorkspaceBuilder::new(workspace_path, USER_AGENT).init()?; + pub fn init() -> Result { + let env_workspace_path = ::std::env::var("CRATESFYI_RUSTWIDE_WORKSPACE"); + let workspace_path = env_workspace_path + .as_ref() + .map(|v| v.as_str()) + .unwrap_or(DEFAULT_RUSTWIDE_WORKSPACE); + let workspace = WorkspaceBuilder::new(Path::new(workspace_path), USER_AGENT).init()?; workspace.purge_all_build_dirs()?; let toolchain = Toolchain::Dist { @@ -306,7 +313,12 @@ impl RustwideBuilder { Ok(res.successful) } - fn execute_build(&self, target: Option<&str>, build: &Build, limits: &Limits) -> Result { + fn execute_build( + &self, + target: Option<&str>, + build: &Build, + limits: &Limits, + ) -> Result { let metadata = Metadata::from_source_dir(&build.host_source_dir())?; let cargo_metadata = CargoMetadata::load(&self.workspace, &self.toolchain, &build.host_source_dir())?; diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 030c1a8f9..435e37a5e 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -9,7 +9,7 @@ use std::process::exit; use std::fs::File; use std::io::Write; use std::time::Duration; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use libc::fork; use time; use docbuilder::RustwideBuilder; @@ -86,7 +86,7 @@ pub fn start_daemon() { QueueInProgress(usize), } - let mut builder = RustwideBuilder::init(&Path::new("rustwide")).unwrap(); + let mut builder = RustwideBuilder::init().unwrap(); let mut status = BuilderState::Fresh; From fd831216b69d32d8b6235b9233919a8a92789728 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 13 Sep 2019 15:53:32 +0200 Subject: [PATCH 18/23] show which environment variable is missing in the daemon --- src/utils/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 435e37a5e..59e594d70 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -27,7 +27,7 @@ pub fn start_daemon() { "CRATESFYI_GITHUB_USERNAME", "CRATESFYI_GITHUB_ACCESSTOKEN"] .iter() { - env::var(v).expect("Environment variable not found"); + env::var(v).expect(&format!("Environment variable {} not found", v)); } let dbopts = opts(); From 9e466a10d43b2b4e677493c28d61893cd5dad447 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 13 Sep 2019 16:02:11 +0200 Subject: [PATCH 19/23] try to update to the latest nightly on every build --- src/docbuilder/rustwide_builder.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 77843568b..0bd1bba6e 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -77,13 +77,11 @@ impl RustwideBuilder { name: "nightly".into(), }; - let mut builder = RustwideBuilder { + Ok(RustwideBuilder { workspace, toolchain, rustc_version: String::new(), - }; - builder.update_toolchain()?; - Ok(builder) + }) } fn update_toolchain(&mut self) -> Result<()> { @@ -215,6 +213,9 @@ impl RustwideBuilder { if !doc_builder.should_build(name, version) { return Ok(false); } + + self.update_toolchain()?; + info!("building package {} {}", name, version); let conn = connect_db()?; From b9a2e28d2bc196927560c69116df3432f640fe9f Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 16 Sep 2019 10:28:24 +0200 Subject: [PATCH 20/23] compare parsed semver when adding a new version --- src/db/add_package.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 6f06daa93..07222a633 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -13,7 +13,7 @@ use rustc_serialize::json::{Json, ToJson}; use slug::slugify; use reqwest::Client; use reqwest::header::ACCEPT; -use semver; +use semver::Version; use postgres::Connection; use time; use error::Result; @@ -153,6 +153,7 @@ pub(crate) fn add_package_into_database(conn: &Connection, // Update versions { + let metadata_version = Version::parse(&metadata_pkg.version)?; let mut versions: Json = try!(conn.query("SELECT versions FROM crates WHERE id = $1", &[&crate_id])) .get(0) @@ -160,7 +161,8 @@ pub(crate) fn add_package_into_database(conn: &Connection, if let Some(versions_array) = versions.as_array_mut() { let mut found = false; for version in versions_array.clone() { - if version.as_string().unwrap() == metadata_pkg.version { + let version = Version::parse(version.as_string().unwrap())?; + if version != metadata_version { found = true; } } From c61f3a88d44c66a40b8ab6b0f9eb2ad67b653223 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 16 Sep 2019 10:29:43 +0200 Subject: [PATCH 21/23] inline limits constants directly into the default impl --- src/docbuilder/limits.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/docbuilder/limits.rs b/src/docbuilder/limits.rs index 9ba9d2b5b..10a28734d 100644 --- a/src/docbuilder/limits.rs +++ b/src/docbuilder/limits.rs @@ -3,11 +3,6 @@ use postgres::Connection; use std::collections::BTreeMap; use std::time::Duration; -const DEFAULT_MEMORY_LIMIT: usize = 3 * 1024 * 1024 * 1024; // 3 GB -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(15 * 60); // 15 minutes -const DEFAULT_NETWORKING: bool = false; -const DEFAULT_MAX_LOG_SIZE: usize = 100 * 1024; // 100 KB - pub(crate) struct Limits { memory: usize, timeout: Duration, @@ -18,10 +13,10 @@ pub(crate) struct Limits { impl Default for Limits { fn default() -> Self { Self { - memory: DEFAULT_MEMORY_LIMIT, - timeout: DEFAULT_TIMEOUT, - networking: DEFAULT_NETWORKING, - max_log_size: DEFAULT_MAX_LOG_SIZE, + memory: 3 * 1024 * 1024 * 1024, // 3 GB + timeout: Duration::from_secs(15 * 60), // 15 minutes + networking: false, + max_log_size: 100 * 1024, // 100 KB } } } From 6043ff993006a2e23a3645a4ff118f0865f54363 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 23 Sep 2019 10:01:29 +0200 Subject: [PATCH 22/23] upgrade rustwide to 0.3.0 --- Cargo.lock | 158 +++++++++++-------- Cargo.toml | 2 +- src/docbuilder/rustwide_builder.rs | 233 +++++++++++++++-------------- 3 files changed, 214 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d3e68a2b..63b42cf7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,7 +74,7 @@ name = "atty" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -91,7 +91,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -102,7 +102,7 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -131,6 +131,14 @@ dependencies = [ "safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bitflags" version = "0.7.0" @@ -248,7 +256,7 @@ dependencies = [ "jobserver 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libgit2-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -290,7 +298,7 @@ name = "chrono" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", @@ -332,7 +340,7 @@ name = "commoncrypto-sys" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -369,7 +377,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -378,7 +386,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -386,7 +394,7 @@ name = "core-foundation-sys" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -436,7 +444,7 @@ dependencies = [ "html5ever 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)", "iron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "magic 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", "params 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -451,7 +459,7 @@ dependencies = [ "rusoto_credential 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", "rusoto_s3 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", - "rustwide 0.2.0 (git+https://github.com/rust-lang/rustwide)", + "rustwide 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "sass-rs 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama_postgres 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -546,7 +554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "curl-sys 0.4.18 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", "schannel 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", @@ -560,7 +568,7 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libnghttp2-sys 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", @@ -582,7 +590,7 @@ name = "dirs" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -667,7 +675,7 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -677,7 +685,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -686,7 +694,7 @@ name = "flate2" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -714,7 +722,7 @@ name = "fs2" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -777,13 +785,23 @@ dependencies = [ "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "getrandom" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", + "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "git2" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libgit2-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1029,7 +1047,7 @@ name = "iovec" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1065,7 +1083,7 @@ name = "jobserver" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1101,7 +1119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.43" +version = "0.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1120,7 +1138,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libssh2-sys 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1141,7 +1159,7 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1154,7 +1172,7 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1195,7 +1213,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "magic-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1204,7 +1222,7 @@ name = "magic-sys" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1237,7 +1255,7 @@ name = "memchr" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1245,7 +1263,7 @@ name = "memchr" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1302,7 +1320,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1315,7 +1333,7 @@ dependencies = [ "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1340,7 +1358,7 @@ version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1398,7 +1416,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "openssl 0.10.13 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1414,7 +1432,7 @@ version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1434,7 +1452,7 @@ dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1530,7 +1548,7 @@ name = "num_cpus" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1550,7 +1568,7 @@ dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1566,7 +1584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1616,7 +1634,7 @@ name = "parking_lot_core" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1839,7 +1857,7 @@ version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1849,7 +1867,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1860,7 +1878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1871,7 +1889,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1931,7 +1949,7 @@ name = "rand_jitter" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1943,7 +1961,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2193,13 +2211,15 @@ dependencies = [ [[package]] name = "rustwide" -version = "0.2.0" -source = "git+https://github.com/rust-lang/rustwide#fa1480bdedd4fd3f242e4f669839064ba088bf08" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "getrandom 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2246,7 +2266,7 @@ name = "sass-rs" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "sass-sys 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2256,7 +2276,7 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2316,7 +2336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2326,7 +2346,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2437,7 +2457,7 @@ name = "signal-hook" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "signal-hook-registry 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2447,7 +2467,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "arc-swap 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2490,7 +2510,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2640,7 +2660,7 @@ dependencies = [ "bytesize 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2652,7 +2672,7 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2672,7 +2692,7 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2702,7 +2722,7 @@ name = "termion" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2728,7 +2748,7 @@ name = "time" version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2807,7 +2827,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2839,7 +2859,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", "signal-hook 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2909,7 +2929,7 @@ dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3159,6 +3179,11 @@ dependencies = [ "try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "wasi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "winapi" version = "0.2.8" @@ -3219,7 +3244,7 @@ name = "xattr" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3245,6 +3270,7 @@ dependencies = [ "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" "checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" "checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" +"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" "checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" "checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" @@ -3314,6 +3340,7 @@ dependencies = [ "checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" "checksum fwdansi 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "34dd4c507af68d37ffef962063dfa1944ce0dd4d5b82043dbab1dabe088610c3" "checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" +"checksum getrandom 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "473a1265acc8ff1e808cd0a1af8cee3c2ee5200916058a2ca113c29f2d903571" "checksum git2 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "924b2e7d2986e625dcad89e8a429a7b3adee3c3d71e585f4a66c4f7e78715e31" "checksum git2-curl 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f21f0550fd5d3f7c5adb94797fcd3d1002d7fc1fa349c82fe44f3c97ef80b62c" "checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" @@ -3346,7 +3373,7 @@ dependencies = [ "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1" "checksum lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddba4c30a78328befecec92fc94970e53b3ae385827d28620f0f5bb2493081e0" -"checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" +"checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" "checksum libflate 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "21138fc6669f438ed7ae3559d5789a5f0ba32f28c1f0608d1e452b0bb06ee936" "checksum libgit2-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "941a41e23f77323b8c9d2ee118aec9ee39dfc176078c18b4757d3bad049d9ff7" "checksum libnghttp2-sys 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ffbfb81475cc9f625e44f3a8f8b9cf7173815ae1c7cc2fa91853ec009e38198" @@ -3462,7 +3489,7 @@ dependencies = [ "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustfix 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "af7c21531a91512a4a51b490be6ba1c8eff34fdda0dc5bf87dc28d86748aac56" "checksum rusttype 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b8eb11f5b0a98c8eca2fb1483f42646d8c340e83e46ab416f8a063a0fd0eeb20" -"checksum rustwide 0.2.0 (git+https://github.com/rust-lang/rustwide)" = "" +"checksum rustwide 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d62fdc88c46d136aea71a9bd56503cb7570f8e11086d46ddd97a53a45ab8612c" "checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" @@ -3575,6 +3602,7 @@ dependencies = [ "checksum walkdir 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c66c0b9792f0a765345452775f3adbd28dde9d33f30d13e5dcc5ae17cf6f3780" "checksum walkdir 2.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c7904a7e2bb3cdf0cf5e783f44204a85a37a93151738fa349f06680f59a98b45" "checksum want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "797464475f30ddb8830cc529aaaae648d581f99e2036a928877dfde027ddf6b3" +"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/Cargo.toml b/Cargo.toml index 563550bd6..aebb4e804 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ tokio = "0.1" systemstat = "0.1.4" prometheus = { version = "0.7.0", default-features = false } lazy_static = "1.0.0" -rustwide = { git = "https://github.com/rust-lang/rustwide" } +rustwide = "0.3.0" tempdir = "0.3" # iron dependencies diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 0bd1bba6e..7218337b0 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -138,47 +138,49 @@ impl RustwideBuilder { .memory_limit(Some(limits.memory())) .enable_networking(limits.networking()); - build_dir.build(&self.toolchain, &krate, sandbox, |build| { - let res = self.execute_build(None, build, &limits)?; - if !res.successful { - bail!("failed to build dummy crate for {}", self.rustc_version); - } - - info!("copying essential files for {}", self.rustc_version); - let source = build.host_target_dir().join(&res.target).join("doc"); - let dest = ::tempdir::TempDir::new("essential-files")?; - - let files = ESSENTIAL_FILES_VERSIONED - .iter() - .map(|f| (f, true)) - .chain(ESSENTIAL_FILES_UNVERSIONED.iter().map(|f| (f, false))); - for (file, versioned) in files { - let segments = file.rsplitn(2, '.').collect::>(); - let file_name = if versioned { - format!("{}-{}.{}", segments[1], rustc_version, segments[0]) - } else { - file.to_string() - }; - let source_path = source.join(&file_name); - let dest_path = dest.path().join(&file_name); - ::std::fs::copy(&source_path, &dest_path).with_context(|_| { - format!( - "couldn't copy '{}' to '{}'", - source_path.display(), - dest_path.display() - ) - })?; - } - - add_path_into_database(&conn, "", &dest)?; - conn.query( - "INSERT INTO config (name, value) VALUES ('rustc_version', $1) \ - ON CONFLICT (name) DO UPDATE SET value = $1;", - &[&self.rustc_version.to_json()], - )?; - - Ok(()) - })?; + build_dir + .build(&self.toolchain, &krate, sandbox) + .run(|build| { + let res = self.execute_build(None, build, &limits)?; + if !res.successful { + bail!("failed to build dummy crate for {}", self.rustc_version); + } + + info!("copying essential files for {}", self.rustc_version); + let source = build.host_target_dir().join(&res.target).join("doc"); + let dest = ::tempdir::TempDir::new("essential-files")?; + + let files = ESSENTIAL_FILES_VERSIONED + .iter() + .map(|f| (f, true)) + .chain(ESSENTIAL_FILES_UNVERSIONED.iter().map(|f| (f, false))); + for (file, versioned) in files { + let segments = file.rsplitn(2, '.').collect::>(); + let file_name = if versioned { + format!("{}-{}.{}", segments[1], rustc_version, segments[0]) + } else { + file.to_string() + }; + let source_path = source.join(&file_name); + let dest_path = dest.path().join(&file_name); + ::std::fs::copy(&source_path, &dest_path).with_context(|_| { + format!( + "couldn't copy '{}' to '{}'", + source_path.display(), + dest_path.display() + ) + })?; + } + + add_path_into_database(&conn, "", &dest)?; + conn.query( + "INSERT INTO config (name, value) VALUES ('rustc_version', $1) \ + ON CONFLICT (name) DO UPDATE SET value = $1;", + &[&self.rustc_version.to_json()], + )?; + + Ok(()) + })?; build_dir.purge()?; krate.purge_from_cache(&self.workspace)?; @@ -231,83 +233,88 @@ impl RustwideBuilder { .memory_limit(Some(limits.memory())) .enable_networking(limits.networking()); - let res = build_dir.build(&self.toolchain, &krate, sandbox, |build| { - let mut files_list = None; - let mut has_docs = false; - let mut successful_targets = Vec::new(); - - // Do an initial build and then copy the sources in the database - let res = self.execute_build(None, &build, &limits)?; - if res.successful { - debug!("adding sources into database"); - let prefix = format!("sources/{}/{}", name, version); - files_list = Some(add_path_into_database( - &conn, - &prefix, - build.host_source_dir(), - )?); - - has_docs = build - .host_target_dir() - .join(&res.target) - .join("doc") - .join(name.replace("-", "_")) - .is_dir(); - } - - if has_docs { - debug!("adding documentation for the default target to the database"); - self.copy_docs( - &doc_builder, - &build.host_target_dir(), - name, - version, - &res.target, - true, - )?; + let res = build_dir + .build(&self.toolchain, &krate, sandbox) + .run(|build| { + let mut files_list = None; + let mut has_docs = false; + let mut successful_targets = Vec::new(); + + // Do an initial build and then copy the sources in the database + let res = self.execute_build(None, &build, &limits)?; + if res.successful { + debug!("adding sources into database"); + let prefix = format!("sources/{}/{}", name, version); + files_list = Some(add_path_into_database( + &conn, + &prefix, + build.host_source_dir(), + )?); + + has_docs = build + .host_target_dir() + .join(&res.target) + .join("doc") + .join(name.replace("-", "_")) + .is_dir(); + } - // Then build the documentation for all the targets - for target in TARGETS { - debug!("building package {} {} for {}", name, version, target); - let target_res = self.execute_build(Some(target), &build, &limits)?; - if target_res.successful { - // Cargo is not giving any error and not generating documentation of some crates - // when we use a target compile options. Check documentation exists before - // adding target to successfully_targets. - if build.host_target_dir().join(target).join("doc").is_dir() { - debug!("adding documentation for target {} to the database", target); - self.copy_docs( - &doc_builder, - &build.host_target_dir(), - name, - version, - target, - false, - )?; - successful_targets.push(target.to_string()); + if has_docs { + debug!("adding documentation for the default target to the database"); + self.copy_docs( + &doc_builder, + &build.host_target_dir(), + name, + version, + &res.target, + true, + )?; + + // Then build the documentation for all the targets + for target in TARGETS { + debug!("building package {} {} for {}", name, version, target); + let target_res = self.execute_build(Some(target), &build, &limits)?; + if target_res.successful { + // Cargo is not giving any error and not generating documentation of some crates + // when we use a target compile options. Check documentation exists before + // adding target to successfully_targets. + if build.host_target_dir().join(target).join("doc").is_dir() { + debug!( + "adding documentation for target {} to the database", + target + ); + self.copy_docs( + &doc_builder, + &build.host_target_dir(), + name, + version, + target, + false, + )?; + successful_targets.push(target.to_string()); + } } } + + self.upload_docs(doc_builder, &conn, name, version)?; } - self.upload_docs(doc_builder, &conn, name, version)?; - } - - let has_examples = build.host_source_dir().join("examples").is_dir(); - let release_id = add_package_into_database( - &conn, - res.cargo_metadata.root(), - &build.host_source_dir(), - &res, - files_list, - successful_targets, - has_docs, - has_examples, - )?; - add_build_into_database(&conn, &release_id, &res)?; - - doc_builder.add_to_cache(name, version); - Ok(res) - })?; + let has_examples = build.host_source_dir().join("examples").is_dir(); + let release_id = add_package_into_database( + &conn, + res.cargo_metadata.root(), + &build.host_source_dir(), + &res, + files_list, + successful_targets, + has_docs, + has_examples, + )?; + add_build_into_database(&conn, &release_id, &res)?; + + doc_builder.add_to_cache(name, version); + Ok(res) + })?; build_dir.purge()?; krate.purge_from_cache(&self.workspace)?; From 50273431885e834a2cf0fc432e6f258c1a7af1c4 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Sep 2019 17:09:28 +0200 Subject: [PATCH 23/23] simplify option clone --- src/db/add_package.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 07222a633..08dc83fd6 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -216,7 +216,7 @@ fn convert_dependencies(pkg: &MetadataPackage) -> Vec<(String, String, String)> for dependency in &pkg.dependencies { let name = dependency.name.clone(); let version = dependency.req.clone(); - let kind = dependency.kind.as_ref().map(|s| s.clone()).unwrap_or_else(|| "normal".into()); + let kind = dependency.kind.clone().unwrap_or_else(|| "normal".into()); dependencies.push((name, version, kind.to_string())); } dependencies