Merge pull request #460 from jyn514/build-local

Build local packages

Author: pietroalbini

README.md

@@ -105,6 +105,11 @@ docker-compose run web build crate <CRATE_NAME> <CRATE_VERSION>
 # Builds every crate and adds them into database
 # (beware: this may take months to finish)
 docker-compose run web build world
+
+# Builds a local package you have at <SOURCE> and adds it to the database.
+# The package does not have to be on crates.io.
+# The package must be on the local filesystem, git urls are not allowed.
+docker-compose run -v "$(realpath <SOURCE>)":/build web build crate --local /build
 ```
 
 

src/bin/cratesfyi.rs

@@ -9,7 +9,7 @@ extern crate rustwide;
 
 
 use std::env;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use clap::{Arg, App, SubCommand};
 use cratesfyi::{DocBuilder, RustwideBuilder, DocBuilderOptions, db};
@@ -53,12 +53,18 @@ pub fn main() {
                 .about("Builds documentation for a crate")
                 .arg(Arg::with_name("CRATE_NAME")
                     .index(1)
-                    .required(true)
+                    .required_unless("LOCAL")
+                    .requires("CRATE_VERSION")
                     .help("Crate name"))
                 .arg(Arg::with_name("CRATE_VERSION")
                     .index(2)
-                    .required(true)
-                    .help("Version of crate")))
+                    .help("Version of crate"))
+                .arg(Arg::with_name("LOCAL")
+                    .short("-l")
+                    .long("--local")
+                    .takes_value(true)
+                    .conflicts_with_all(&["CRATE_NAME", "CRATE_VERSION"])
+                    .help("Build a crate at a specific path")))
             .subcommand(SubCommand::with_name("add-essential-files")
                 .about("Adds essential files for rustc"))
             .subcommand(SubCommand::with_name("lock").about("Locks cratesfyi daemon to stop \
@@ -155,10 +161,12 @@ pub fn main() {
             docbuilder.save_cache().expect("Failed to save cache");
         } else if let Some(matches) = matches.subcommand_matches("crate") {
             docbuilder.load_cache().expect("Failed to load cache");
-            let mut builder = RustwideBuilder::init().unwrap();
-            builder.build_package(&mut docbuilder, matches.value_of("CRATE_NAME").unwrap(),
-                                  matches.value_of("CRATE_VERSION").unwrap())
-                .expect("Building documentation failed");
+            let mut builder = RustwideBuilder::init().expect("failed to initialize rustwide");
+            match matches.value_of("LOCAL") {
+                Some(path) => builder.build_local_package(&mut docbuilder, Path::new(path)),
+                None => builder.build_package(&mut docbuilder,
+                    matches.value_of("CRATE_NAME").unwrap(), matches.value_of("CRATE_VERSION").unwrap(), None),
+            }.expect("Building documentation failed");
             docbuilder.save_cache().expect("Failed to save cache");
         } else if let Some(_) = matches.subcommand_matches("add-essential-files") {
             let mut builder = RustwideBuilder::init().unwrap();

src/db/add_package.rs

@@ -288,7 +288,7 @@ fn read_rust_doc(file_path: &Path) -> Result<Option<String>> {
 /// Get release_time, yanked and downloads from crates.io
 fn get_release_time_yanked_downloads(
     pkg: &MetadataPackage,
-) -> Result<(Option<time::Timespec>, Option<bool>, Option<i32>)> {
+) -> Result<(time::Timespec, bool, i32)> {
     let url = format!("https://crates.io/api/v1/crates/{}/versions", pkg.name);
     // FIXME: There is probably better way to do this
     //        and so many unwraps...
@@ -332,7 +332,7 @@ fn get_release_time_yanked_downloads(
         }
     }
 
-    Ok((release_time, yanked, downloads))
+    Ok((release_time.unwrap_or(time::get_time()), yanked.unwrap_or(false), downloads.unwrap_or(0)))
 }
 
 

src/db/migrate.rs

@@ -193,6 +193,16 @@ pub fn migrate(version: Option<Version>) -> CratesfyiResult<()> {
             // downgrade query
             "DROP TABLE sandbox_overrides;"
         ),
+        migration!(
+            4,
+            "Make more fields not null",
+            "ALTER TABLE releases ALTER COLUMN release_time SET NOT NULL,
+                                  ALTER COLUMN yanked SET NOT NULL,
+                                  ALTER COLUMN downloads SET NOT NULL",
+            "ALTER TABLE releases ALTER COLUMN release_time DROP NOT NULL,
+                                  ALTER COLUMN yanked DROP NOT NULL,
+                                  ALTER COLUMN downloads DROP NOT NULL"
+        )
     ];
 
     for migration in migrations {

src/docbuilder/queue.rs

@@ -59,7 +59,7 @@ impl DocBuilder {
         let name: String = query.get(0).get(1);
         let version: String = query.get(0).get(2);
 
-        match builder.build_package(self, &name, &version) {
+        match builder.build_package(self, &name, &version, None) {
             Ok(_) => {
                 let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]);
                 ::web::metrics::TOTAL_BUILDS.inc();

src/docbuilder/rustwide_builder.rs

@@ -204,7 +204,7 @@ impl RustwideBuilder {
         crates_from_path(
             &doc_builder.options().crates_io_index_path.clone(),
             &mut |name, version| {
-                match self.build_package(doc_builder, name, version) {
+                match self.build_package(doc_builder, name, version, None) {
                     Ok(status) => {
                         count += 1;
                         if status && count % 10 == 0 {
@@ -218,11 +218,23 @@ impl RustwideBuilder {
         )
     }
 
+    pub fn build_local_package(
+        &mut self,
+        doc_builder: &mut DocBuilder,
+        path: &Path,
+    ) -> Result<bool> {
+        self.update_toolchain()?;
+        let metadata = CargoMetadata::load(&self.workspace, &self.toolchain, path)?;
+        let package = metadata.root();
+        self.build_package(doc_builder, &package.name, &package.version, Some(path))
+    }
+
     pub fn build_package(
         &mut self,
         doc_builder: &mut DocBuilder,
         name: &str,
         version: &str,
+        local: Option<&Path>,
     ) -> Result<bool> {
         if !doc_builder.should_build(name, version) {
             return Ok(false);
@@ -238,7 +250,11 @@ impl RustwideBuilder {
         let mut build_dir = self.workspace.build_dir(&format!("{}-{}", name, version));
         build_dir.purge()?;
 
-        let krate = Crate::crates_io(name, version);
+        let krate = if let Some(path) = local {
+            Crate::local(path)
+        } else {
+            Crate::crates_io(name, version)
+        };
         krate.fetch(&self.workspace)?;
 
         let sandbox = SandboxBuilder::new()