From cd3acd231460bdf51db0881879d25d5aa371e83c Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 10 Oct 2022 11:03:11 +0100 Subject: [PATCH 1/6] Add support for alternative crate registries --- CHANGELOG.md | 17 +++++- src/async_client.rs | 25 ++++---- src/helper.rs | 139 ++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 17 ++++++ src/sync_client.rs | 17 +++--- src/types.rs | 12 +++- 6 files changed, 206 insertions(+), 21 deletions(-) create mode 100644 src/helper.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 67bcc79..cfaa1ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 0.9.0 + +This version has a breaking change to support specifying alternative crate registries. + +### Features + +* Query registries other than crates-io + +### (Breaking) Changes + +* `AsyncClient` and `SyncClient` take an `Option(&Registry)` +* Types, make field optional: User {url} + ## 0.8.1 * Add `AsyncClient::with_http_client` constructor @@ -10,7 +23,7 @@ ## 0.8.0 - 2022-01-29 -This version has quite a few breaking changes, +This version has quite a few breaking changes, mainly to clean up and future-proof the API. ### Features @@ -95,7 +108,7 @@ mainly to clean up and future-proof the API. * Crate {recent_downloads, exact_match} * CrateResponse {versions, keywords, categories} * Version {crate_size, published_by} -* Make field optional: User {kind} +* Make field optional: User {kind} * Fix getting the reverse dependencies. * Rearrange the received data for simpler manipulation. * Add 3 new types: diff --git a/src/async_client.rs b/src/async_client.rs index 30cf273..dcac95f 100644 --- a/src/async_client.rs +++ b/src/async_client.rs @@ -1,14 +1,14 @@ use futures::future::BoxFuture; use futures::prelude::*; use futures::{future::try_join_all, try_join}; -use reqwest::{header, Client as HttpClient, StatusCode, Url}; +use reqwest::{Client as HttpClient, StatusCode, Url}; use serde::de::DeserializeOwned; use std::collections::VecDeque; use super::Error; use crate::error::JsonDecodeError; -use crate::types::*; +use crate::{helper::*, types::*}; /// Asynchronous client for the crates.io API. #[derive(Clone)] @@ -116,6 +116,7 @@ impl Client { /// let client = crates_io_api::AsyncClient::new( /// "my_bot (help@my_bot.com)", /// std::time::Duration::from_millis(1000), + /// None, /// ).unwrap(); /// # Ok(()) /// # } @@ -123,19 +124,16 @@ impl Client { pub fn new( user_agent: &str, rate_limit: std::time::Duration, + registry: Option<&Registry>, ) -> Result { - let mut headers = header::HeaderMap::new(); - headers.insert( - header::USER_AGENT, - header::HeaderValue::from_str(user_agent)?, - ); + let headers = setup_headers(user_agent, registry)?; let client = HttpClient::builder() .default_headers(headers) .build() .unwrap(); - Ok(Self::with_http_client(client, rate_limit)) + Ok(Self::with_http_client(client, rate_limit, registry)) } /// Instantiate a new client. @@ -146,14 +144,20 @@ impl Client { /// At most one request will be executed in the specified duration. /// The guidelines suggest 1 per second or less. /// (Only one request is executed concurrenly, even if the given Duration is 0). - pub fn with_http_client(client: HttpClient, rate_limit: std::time::Duration) -> Self { + pub fn with_http_client( + client: HttpClient, + rate_limit: std::time::Duration, + registry: Option<&Registry>, + ) -> Self { let limiter = std::sync::Arc::new(tokio::sync::Mutex::new(None)); + let base_url = base_url(registry); + Self { rate_limit, last_request_time: limiter, client, - base_url: Url::parse("https://crates.io/api/v1/").unwrap(), + base_url: Url::parse(base_url).unwrap(), } } @@ -485,6 +489,7 @@ mod test { Client::new( "crates-io-api-continuous-integration (github.com/theduke/crates-io-api)", std::time::Duration::from_millis(1000), + None, ) .unwrap() } diff --git a/src/helper.rs b/src/helper.rs new file mode 100644 index 0000000..f96a91b --- /dev/null +++ b/src/helper.rs @@ -0,0 +1,139 @@ +//! Helper functions for querying crate registries +//! +//! +use crate::{error::Error, types::*}; + +use reqwest::header; +use std::env; + +/// Setup the headers for a sync or async request +pub fn setup_headers( + user_agent: &str, + registry: Option<&Registry>, +) -> Result { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::USER_AGENT, + header::HeaderValue::from_str(user_agent)?, + ); + + if registry.is_some() { + match ®istry.unwrap().name { + Some(name) => { + match env::var(format!("CARGO_REGISTRIES_{}_TOKEN", name.to_uppercase())) { + Ok(foo) => { + headers.insert(header::AUTHORIZATION, header::HeaderValue::from_str(&foo)?); + () + } + _ => (), + } + } + None => match ®istry.unwrap().token { + Some(token) => { + headers.insert( + header::AUTHORIZATION, + header::HeaderValue::from_str(&token)?, + ); + } + None => (), + }, + } + } + Ok(headers) +} + +/// Determine the url of the crate registry being queried. +pub fn base_url(registry: Option<&Registry>) -> &str { + match registry { + Some(reg) => reg.url.as_str(), + None => "https://crates.io/api/v1/", + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_base_url_default() -> Result<(), Error> { + assert_eq!(base_url(None), "https://crates.io/api/v1/"); + Ok(()) + } + + #[test] + fn test_base_url_private() -> Result<(), Error> { + let reg = &Registry { + url: "https://crates.foobar.com/api/v1/".to_string(), + name: None, + token: None, + }; + assert_eq!(base_url(Some(reg)), "https://crates.foobar.com/api/v1/"); + Ok(()) + } + + #[test] + fn test_crates_io_headers() -> Result<(), Error> { + let reg = None; + let user_agent = "crates-io-api-continuous-integration (github.com/theduke/crates-io-api)"; + let headers = setup_headers(user_agent, reg).unwrap(); + + let mut exp_headers = header::HeaderMap::new(); + exp_headers.insert( + header::USER_AGENT, + header::HeaderValue::from_str(user_agent).unwrap(), + ); + + assert_eq!(headers, exp_headers); + Ok(()) + } + + #[test] + fn test_private_registry_name_headers() -> Result<(), Error> { + let reg = &Registry { + url: "https://crates.foobar.com/api/v1/".to_string(), + name: Some("foobar".to_string()), + token: None, + }; + env::set_var("CARGO_REGISTRIES_FOOBAR_TOKEN", "baz"); + let user_agent = "crates-io-api-continuous-integration (github.com/theduke/crates-io-api)"; + let headers = setup_headers(user_agent, Some(reg)).unwrap(); + + let mut exp_headers = header::HeaderMap::new(); + exp_headers.insert( + header::USER_AGENT, + header::HeaderValue::from_str(user_agent).unwrap(), + ); + exp_headers.insert( + header::AUTHORIZATION, + header::HeaderValue::from_str("baz").unwrap(), + ); + + assert_eq!(headers, exp_headers); + Ok(()) + } + + #[test] + fn test_private_registry_token_headers() -> Result<(), Error> { + let reg = &Registry { + url: "https://crates.foobar.com/api/v1/".to_string(), + name: None, + token: Some("foobar".to_string()), + }; + env::set_var("CARGO_REGISTRIES_FOOBAR_TOKEN", "baz"); + let user_agent = "crates-io-api-continuous-integration (github.com/theduke/crates-io-api)"; + let headers = setup_headers(user_agent, Some(reg)).unwrap(); + + let mut exp_headers = header::HeaderMap::new(); + exp_headers.insert( + header::USER_AGENT, + header::HeaderValue::from_str(user_agent).unwrap(), + ); + exp_headers.insert( + header::AUTHORIZATION, + header::HeaderValue::from_str("foobar").unwrap(), + ); + + assert_eq!(headers, exp_headers); + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index b304da1..57e9bdc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ //! let client = SyncClient::new( //! "my-user-agent (my-contact@domain.com)", //! std::time::Duration::from_millis(1000), +//! None, //! ).unwrap(); //! // Retrieve summary data. //! let summary = client.summary()?; @@ -40,18 +41,34 @@ //! Ok(()) //! } //! ``` +//! Instantiate a client for a private registry with environment variable authentication +//! +//! ```rust +//! use crates_io_api::{SyncClient,Registry}; +//! let client = SyncClient::new( +//! "my-user-agent (my-contact@domain.com)", +//! std::time::Duration::from_millis(1000), +//! Some(&Registry{ +//! url: "https://crates.my-registry.com/api/v1/".to_string(), +//! name: Some("my_registry".to_string()), +//! token: None, +//! }), +//! ).unwrap(); +//! ``` #![recursion_limit = "128"] #![deny(missing_docs)] mod async_client; mod error; +mod helper; mod sync_client; mod types; pub use crate::{ async_client::Client as AsyncClient, error::{Error, NotFoundError, PermissionDeniedError}, + helper::*, sync_client::SyncClient, types::*, }; diff --git a/src/sync_client.rs b/src/sync_client.rs index 5e4535d..28a3944 100644 --- a/src/sync_client.rs +++ b/src/sync_client.rs @@ -2,10 +2,10 @@ use super::*; use std::iter::Extend; use log::trace; -use reqwest::{blocking::Client as HttpClient, header, StatusCode, Url}; +use reqwest::{blocking::Client as HttpClient, StatusCode, Url}; use serde::de::DeserializeOwned; -use crate::{error::JsonDecodeError, types::*}; +use crate::{error::JsonDecodeError, helper::*, types::*}; /// A synchronous client for the crates.io API. pub struct SyncClient { @@ -33,6 +33,7 @@ impl SyncClient { /// let client = crates_io_api::AsyncClient::new( /// "my_bot (help@my_bot.com)", /// std::time::Duration::from_millis(1000), + /// None, /// ).unwrap(); /// # Ok(()) /// # } @@ -40,19 +41,17 @@ impl SyncClient { pub fn new( user_agent: &str, rate_limit: std::time::Duration, + registry: Option<&Registry>, ) -> Result { - let mut headers = header::HeaderMap::new(); - headers.insert( - header::USER_AGENT, - header::HeaderValue::from_str(user_agent)?, - ); + let headers = setup_headers(user_agent, registry)?; + let base_url = base_url(registry); Ok(Self { client: HttpClient::builder() .default_headers(headers) .build() .unwrap(), - base_url: Url::parse("https://crates.io/api/v1/").unwrap(), + base_url: Url::parse(base_url).unwrap(), rate_limit, last_request_time: std::sync::Mutex::new(None), }) @@ -302,6 +301,7 @@ impl SyncClient { /// # let client = SyncClient::new( /// # "my-bot-name (my-contact@domain.com)", /// # std::time::Duration::from_millis(1000), + /// # None, /// # ).unwrap(); /// let q = CratesQuery::builder() /// .sort(Sort::Alphabetical) @@ -334,6 +334,7 @@ mod test { SyncClient::new( "crates-io-api-ci (github.com/theduke/crates-io-api)", std::time::Duration::from_millis(1000), + None, ) .unwrap() } diff --git a/src/types.rs b/src/types.rs index 5dff9e9..297972c 100644 --- a/src/types.rs +++ b/src/types.rs @@ -4,6 +4,16 @@ use chrono::{DateTime, NaiveDate, Utc}; use serde_derive::*; use std::collections::HashMap; +/// Used to specify the registry being queried by either client. +pub struct Registry { + /// Url of the registry + pub url: String, + /// Name of the registry + pub name: Option, + /// Token used to authenticate registry requests. + pub token: Option, +} + /// Used to specify the sort behaviour of the `Client::crates()` method. #[derive(Deserialize, Debug, Clone, PartialEq, Eq)] pub struct ApiErrors { @@ -432,7 +442,7 @@ pub struct User { pub kind: Option, pub login: String, pub name: Option, - pub url: String, + pub url: Option, } /// Additional crate author metadata. From 3e3c3b7676f9e791417d78b7538d553f2923c2bb Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 10 Oct 2022 14:50:17 +0100 Subject: [PATCH 2/6] Markups to support alternative registry --- src/async_client.rs | 27 +++++++++++++++++++++++++-- src/helper.rs | 13 ++++++++----- src/lib.rs | 6 ------ src/sync_client.rs | 25 ++++++++++++++++++++++--- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/src/async_client.rs b/src/async_client.rs index dcac95f..7efabb5 100644 --- a/src/async_client.rs +++ b/src/async_client.rs @@ -116,7 +116,6 @@ impl Client { /// let client = crates_io_api::AsyncClient::new( /// "my_bot (help@my_bot.com)", /// std::time::Duration::from_millis(1000), - /// None, /// ).unwrap(); /// # Ok(()) /// # } @@ -124,6 +123,31 @@ impl Client { pub fn new( user_agent: &str, rate_limit: std::time::Duration, + ) -> Result { + Self::build(user_agent, rate_limit, None) + } + + /// Build a new client. + /// + /// Returns an [`Error`] if the given user agent is invalid. + /// ```rust + /// use crates_io_api::{AsyncClient,Registry}; + /// # fn f() -> Result<(), Box> { + /// let client = crates_io_api::AsyncClient::build( + /// "my_bot (help@my_bot.com)", + /// std::time::Duration::from_millis(1000), + /// Some(&Registry{ + /// url: "https://crates.my-registry.com/api/v1/".to_string(), + /// name: Some("my_registry".to_string()), + /// token: None, + /// }), + /// ).unwrap(); + /// # Ok(()) + /// # } + /// ``` + pub fn build( + user_agent: &str, + rate_limit: std::time::Duration, registry: Option<&Registry>, ) -> Result { let headers = setup_headers(user_agent, registry)?; @@ -489,7 +513,6 @@ mod test { Client::new( "crates-io-api-continuous-integration (github.com/theduke/crates-io-api)", std::time::Duration::from_millis(1000), - None, ) .unwrap() } diff --git a/src/helper.rs b/src/helper.rs index f96a91b..c5f482d 100644 --- a/src/helper.rs +++ b/src/helper.rs @@ -1,7 +1,7 @@ //! Helper functions for querying crate registries //! //! -use crate::{error::Error, types::*}; +use crate::types::*; use reqwest::header; use std::env; @@ -17,8 +17,8 @@ pub fn setup_headers( header::HeaderValue::from_str(user_agent)?, ); - if registry.is_some() { - match ®istry.unwrap().name { + match ®istry { + Some(registry) => match ®istry.name { Some(name) => { match env::var(format!("CARGO_REGISTRIES_{}_TOKEN", name.to_uppercase())) { Ok(foo) => { @@ -28,7 +28,7 @@ pub fn setup_headers( _ => (), } } - None => match ®istry.unwrap().token { + None => match ®istry.token { Some(token) => { headers.insert( header::AUTHORIZATION, @@ -37,8 +37,10 @@ pub fn setup_headers( } None => (), }, - } + }, + None => (), } + Ok(headers) } @@ -53,6 +55,7 @@ pub fn base_url(registry: Option<&Registry>) -> &str { #[cfg(test)] mod test { use super::*; + use crate::Error; #[test] fn test_base_url_default() -> Result<(), Error> { diff --git a/src/lib.rs b/src/lib.rs index 57e9bdc..d77eb98 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,6 @@ //! let client = SyncClient::new( //! "my-user-agent (my-contact@domain.com)", //! std::time::Duration::from_millis(1000), -//! None, //! ).unwrap(); //! // Retrieve summary data. //! let summary = client.summary()?; @@ -48,11 +47,6 @@ //! let client = SyncClient::new( //! "my-user-agent (my-contact@domain.com)", //! std::time::Duration::from_millis(1000), -//! Some(&Registry{ -//! url: "https://crates.my-registry.com/api/v1/".to_string(), -//! name: Some("my_registry".to_string()), -//! token: None, -//! }), //! ).unwrap(); //! ``` diff --git a/src/sync_client.rs b/src/sync_client.rs index 28a3944..e4305e0 100644 --- a/src/sync_client.rs +++ b/src/sync_client.rs @@ -33,7 +33,6 @@ impl SyncClient { /// let client = crates_io_api::AsyncClient::new( /// "my_bot (help@my_bot.com)", /// std::time::Duration::from_millis(1000), - /// None, /// ).unwrap(); /// # Ok(()) /// # } @@ -41,6 +40,28 @@ impl SyncClient { pub fn new( user_agent: &str, rate_limit: std::time::Duration, + ) -> Result { + Self::build(user_agent, rate_limit, None) + } + + /// ```rust + /// use crates_io_api::{SyncClient,Registry}; + /// # fn f() -> Result<(), Box> { + /// let client = crates_io_api::SyncClient::build( + /// "my_bot (help@my_bot.com)", + /// std::time::Duration::from_millis(1000), + /// Some(&Registry{ + /// url: "https://crates.my-registry.com/api/v1/".to_string(), + /// name: Some("my_registry".to_string()), + /// token: None, + /// }), + /// ).unwrap(); + /// # Ok(()) + /// # } + /// ``` + pub fn build( + user_agent: &str, + rate_limit: std::time::Duration, registry: Option<&Registry>, ) -> Result { let headers = setup_headers(user_agent, registry)?; @@ -301,7 +322,6 @@ impl SyncClient { /// # let client = SyncClient::new( /// # "my-bot-name (my-contact@domain.com)", /// # std::time::Duration::from_millis(1000), - /// # None, /// # ).unwrap(); /// let q = CratesQuery::builder() /// .sort(Sort::Alphabetical) @@ -334,7 +354,6 @@ mod test { SyncClient::new( "crates-io-api-ci (github.com/theduke/crates-io-api)", std::time::Duration::from_millis(1000), - None, ) .unwrap() } From fe512aea66d7e94c5db6c6a83654a1597599c7d0 Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 10 Oct 2022 16:29:25 +0100 Subject: [PATCH 3/6] More markups --- CHANGELOG.md | 7 ++----- src/async_client.rs | 8 ++++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfaa1ce..f94344a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,13 @@ # Changelog -## 0.9.0 - -This version has a breaking change to support specifying alternative crate registries. - ### Features * Query registries other than crates-io + - Additional `AsyncClient::build()` and `SyncClient::build()` functions. + For building a client for an alternative registry. ### (Breaking) Changes -* `AsyncClient` and `SyncClient` take an `Option(&Registry)` * Types, make field optional: User {url} ## 0.8.1 diff --git a/src/async_client.rs b/src/async_client.rs index 7efabb5..4f8103e 100644 --- a/src/async_client.rs +++ b/src/async_client.rs @@ -157,7 +157,9 @@ impl Client { .build() .unwrap(); - Ok(Self::with_http_client(client, rate_limit, registry)) + let base_url = base_url(registry); + + Ok(Self::with_http_client(client, rate_limit, base_url)) } /// Instantiate a new client. @@ -171,12 +173,10 @@ impl Client { pub fn with_http_client( client: HttpClient, rate_limit: std::time::Duration, - registry: Option<&Registry>, + base_url: &str, ) -> Self { let limiter = std::sync::Arc::new(tokio::sync::Mutex::new(None)); - let base_url = base_url(registry); - Self { rate_limit, last_request_time: limiter, From cece007de1c7c35062bce5649a6a7ee68dcee22b Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 10 Oct 2022 16:30:33 +0100 Subject: [PATCH 4/6] More markups --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f94344a..070369c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### (Breaking) Changes +* `AsyncClient::with_http_client()` now requires the crate registry url to be specified. * Types, make field optional: User {url} ## 0.8.1 From 64b0a65e7bceb41ea71d3d5c856943e9b40b602b Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 11 Oct 2022 10:04:17 +0100 Subject: [PATCH 5/6] Tidy up helper.rs --- src/helper.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/helper.rs b/src/helper.rs index c5f482d..2b78e41 100644 --- a/src/helper.rs +++ b/src/helper.rs @@ -1,8 +1,6 @@ //! Helper functions for querying crate registries -//! -//! -use crate::types::*; +use crate::types::*; use reqwest::header; use std::env; @@ -20,21 +18,19 @@ pub fn setup_headers( match ®istry { Some(registry) => match ®istry.name { Some(name) => { - match env::var(format!("CARGO_REGISTRIES_{}_TOKEN", name.to_uppercase())) { - Ok(foo) => { - headers.insert(header::AUTHORIZATION, header::HeaderValue::from_str(&foo)?); - () - } - _ => (), - } - } - None => match ®istry.token { - Some(token) => { + if let Ok(token) = + env::var(format!("CARGO_REGISTRIES_{}_TOKEN", name.to_uppercase())) + { headers.insert( header::AUTHORIZATION, header::HeaderValue::from_str(&token)?, ); } + } + None => match ®istry.token { + Some(token) => { + headers.insert(header::AUTHORIZATION, header::HeaderValue::from_str(token)?); + } None => (), }, }, From 2133b87210b3683102f634d952f137f01e10b3f4 Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 11 Oct 2022 10:12:06 +0100 Subject: [PATCH 6/6] Update docstring --- src/async_client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/async_client.rs b/src/async_client.rs index 4f8103e..7182b62 100644 --- a/src/async_client.rs +++ b/src/async_client.rs @@ -162,7 +162,7 @@ impl Client { Ok(Self::with_http_client(client, rate_limit, base_url)) } - /// Instantiate a new client. + /// Instantiate a new client, for the registry sepcified by base_url. /// /// To respect the offical [Crawler Policy](https://crates.io/policies#crawlers), /// you must specify both a descriptive user agent and a rate limit interval.