Improve documentation and clippy.

  - Implemented FromStr properly for Manifest to allow for the use of parse()

Author: terjekv

README.md

@@ -52,3 +52,22 @@ async fn main() {
     }
 }
 ```
+
+## A word about server backends
+
+There are three valid options for backends for a given server. These are:
+
+- `CVMFS`: This backend requires `cvmfs/info/v1/repositories.json` to be present on the server. Scrape fails if it is missing.
+- `S3`: Does not even attempt to fetch `cvmfs/info/v1/repositories.json`. Note that if any server has S3 as a backend a list of repositories *must* be passed to the scraper as there is no other way to determine the list of repositories for S3 servers. Due to the async scraping of all servers, there is currently no support for falling back on repositories detected from other server types (including possibly the Stratum0).
+- `AutoDetect`: This backend Aatempts to fetch `cvmfs/info/v1/repositories.json` but does not fail if it is missing. If the scraper fails to fetch the file, the backend will be assumed to be S3. If the list of repositories is empty, the scraper will return an empty list. If your S3 server has no repositories, setting the backend to AutoDetect will allow the scraper to continue without failing.
+
+For populated servers, the field `backend_detected` will be set to the detected backend, which for explicit S3 or CVMFS servers will be the same as requested type.
+
+## What repositories are scraped?
+
+- For servers that are set to or detected as CVMFS, the scraper will scrape the union of the detected and configurations explicitly stated repositories.
+- For servers that are set to or detected as S3, only the explicitly stated repositories will be scraped (and the scraper will fail if the server type is explicitly set to S3 and no repositories are passed).
+
+## License
+
+Licensed under the MIT license. See the LICENSE file for details.

src/lib.rs

@@ -73,8 +73,8 @@ pub mod errors;
 
 pub use errors::{CVMFSScraperError, HostnameError, ManifestError, ScrapeError};
 pub use models::{
-    FailedServer, Hostname, PopulatedRepositoryOrReplica, PopulatedServer, ScrapedServer, Server,
-    ServerBackendType, ServerType,
+    FailedServer, Hostname, Manifest, MaybeRfc2822DateTime, PopulatedRepositoryOrReplica,
+    PopulatedServer, ScrapedServer, Server, ServerBackendType, ServerType,
 };
 
 use futures::future::join_all;

src/models/cvmfs_published.rs

@@ -8,22 +8,28 @@ use crate::utilities::{parse_boolean_field, parse_hex_field, parse_number_field}
 /// The manifest of a repository or replica.
 ///
 /// The fields are:
-/// - C: Cryptographic hash of the repository’s current root catalog
-/// - B: Size of the root file catalog in bytes
-/// - A: true if the catalog should be fetched under its alternative name
-/// - R: MD5 hash of the repository’s current root path (usually always d41d8cd98f00b204e9800998ecf8427e)
-/// - X: Cryptographic hash of the signing certificate
-/// - G: true if the repository is garbage-collectable
-/// - H: Cryptographic hash of the repository’s named tag history database
-/// - T: Unix timestamp of this particular revision
-/// - D: Time To Live (TTL) of the root catalog
-/// - S: Revision number of this published revision
-/// - N: The full name of the manifested repository
-/// - M: Cryptographic hash of the repository JSON metadata
-/// - Y: Cryptographic hash of the reflog checksum
-/// - L: currently unused (reserved for micro catalogs)
-/// signature - In order to provide authoritative information about a repository publisher, the repository manifest is signed by an X.509 certificate together with its private key.
-
+/// - c: Cryptographic hash of the repository’s current root catalog
+/// - b: Size of the root file catalog in bytes
+/// - a: true if the catalog should be fetched under its alternative name
+/// - r: MD5 hash of the repository’s current root path (usually always d41d8cd98f00b204e9800998ecf8427e)
+/// - x: Cryptographic hash of the signing certificate
+/// - g: true if the repository is garbage-collectable
+/// - h: Cryptographic hash of the repository’s named tag history database
+/// - t: Unix timestamp of this particular revision
+/// - d: Time To Live (TTL) of the root catalog
+/// - s: Revision number of this published revision
+/// - n: The full name of the manifested repository
+/// - m: Cryptographic hash of the repository JSON metadata
+/// - y: Cryptographic hash of the reflog checksum
+/// - l: currently unused (reserved for micro catalogs)
+/// - signature: In order to provide authoritative information about a repository publisher, the
+///              repository manifest is signed by an X.509 certificate together with its private key.
+///              This field is not validated by this library.
+///
+/// Note that the field names are lowercase, but the field names in the manifest itself are uppercase.
+///
+/// See https://cvmfs.readthedocs.io/en/stable/cpt-details.html#repository-manifest-cvmfspublished for
+/// more information.
 #[derive(Deserialize, Serialize, Clone, PartialEq)]
 pub struct Manifest {
     pub c: HexString,
@@ -68,8 +74,10 @@ impl std::fmt::Debug for Manifest {
     }
 }
 
-impl Manifest {
-    pub fn from_str(content: &str) -> Result<Self, ManifestError> {
+impl std::str::FromStr for Manifest {
+    type Err = ManifestError;
+
+    fn from_str(content: &str) -> Result<Self, Self::Err> {
         let mut data: HashMap<char, String> = HashMap::new();
         let mut signature: String = String::new();
         let mut is_signature = false;
@@ -111,7 +119,9 @@ impl Manifest {
 
         Ok(manifest)
     }
+}
 
+impl Manifest {
     pub fn display(&self) {
         println!("  Manifest for repository: {}", self.n);
         println!("    Root catalog hash: {}", self.c);

src/models/generic.rs

@@ -106,6 +106,16 @@ impl<'de> Deserialize<'de> for HexString {
     }
 }
 
+/// A wrapped optional string that may be a RFC 2822 date-time.
+///
+/// Due to the fact that the date-time fields generated into the CVMFS JSON files
+/// are produced with the `date` command, they may be localized to the system
+/// that generated them. This means that the date-time fields may not parsable
+/// with any degree of sanity.
+///
+/// To offer both the option of a time-parsed field and the raw string, we store
+/// the string itself and provide a method (`try_into_datetime`) to attempt to
+/// parse the string into a `DateTime<Utc>`.
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Default)]
 pub struct MaybeRfc2822DateTime(pub Option<String>);
 

src/models/mod.rs

@@ -5,7 +5,8 @@ mod meta_json;
 mod repositories_json;
 mod servers;
 
-pub use generic::{HexString, Hostname};
+pub use cvmfs_published::Manifest;
+pub use generic::{HexString, Hostname, MaybeRfc2822DateTime};
 pub use servers::{
     FailedServer, PopulatedRepositoryOrReplica, PopulatedServer, ScrapedServer, Server,
     ServerBackendType, ServerType,

src/models/servers.rs

@@ -548,10 +548,7 @@ impl RepositoryOrReplica {
             self.server.hostname.0, self.name
         );
         let response = client.get(url).send().await?;
-        let content = response.error_for_status()?.text().await?;
-        let content = content.as_str();
-        // println!("{}", content);
-        Manifest::from_str(content)
+        response.error_for_status()?.text().await?.parse()
     }
 
     async fn fetch_repository_status_json(
@@ -569,6 +566,22 @@ impl RepositoryOrReplica {
     }
 }
 
+/// A populated repository or replica object.
+///
+/// This object represents a CVMFS repository or replica that has been scraped for information about
+/// the repository. For fetching the revision of the repository, one can use the `revision` method
+/// as a shortcut to get the revision from the manifest.
+///
+/// Fields:
+///
+/// - name: The name of the repository
+/// - manifest: The manifest of the repository
+/// - last_snapshot: The last time a snapshot was taken (optional)
+/// - last_gc: The last time garbage collection was run (optional)
+///
+/// The MaybeRfc2822DateTime type is used to represent a date and time that may or may not be present,
+/// and may or may not be in the RFC 2822 format. See the documentation for the MaybeRfc2822DateTime
+/// type for more information.
 #[derive(Debug, Serialize, Clone, PartialEq)]
 pub struct PopulatedRepositoryOrReplica {
     pub name: String,