v0.0.6:

  - Adds support to only scrape repos in `forced_repositories`.
  - Updates depdencies.

Author: terjekv

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "cvmfs_server_scraper"
-version = "0.0.5"
+version = "0.0.6"
 edition = "2021"
 authors = ["Terje Kvernes <[email protected]>"]
 license = "MIT"
@@ -15,7 +15,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 reqwest = { version = "0", features = ["json"] }
 tokio = { version = "1", features = ["full"] }
-thiserror = "1"
+thiserror = "2"
 semver = "1"
 chrono = { version = "0.4", features = ["serde"] }
 yare = "3"

Cargo.toml

@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this will adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) once
 we reach version 0.1.0, up until then, expect breaking changes.
 
+## [0.0.6] - 2025-10-20
+
+### Added
+
+- `scrape` for a server now takes a boolean argument to indicate if only the explicitly listed repositories for that server are to be scraped, overriding `ignored_repositories`.
+  This parameter is also added to the `scrape_servers` API, in both cases requiring consumers to update their code accordingly. To retain previous behavior, pass `false` to
+  either function. If using the builder interface, `only_scrape_forced_repositories(true|false)` is available. The default is `false`, retaining previous behavior and requiring no changes.
+
 ## [0.0.5] - 2024-10-18
 
 ### Added

Changelog.md

@@ -42,6 +42,7 @@ async fn main() -> Result<(), CVMFSScraperError> {
     let scraped_servers = Scraper::new()
        .forced_repositories(repolist)
        .ignored_repositories(ignored_repos)
+       .only_scrape_forced_repositories(false) // Only scrape forced repositories if true, overrides ignored_repositories
        .geoapi_servers(DEFAULT_GEOAPI_SERVERS.clone())? // This is the default list
        .with_servers(servers) // Transitions to a WithServer state.
        .validate()? // Transitions to a ValidatedAndReady state, now immutable.
@@ -74,6 +75,8 @@ For populated servers, the field `backend_detected` will be set to the detected
 
 ## What repositories are scraped?
 
+If `only_scrape_forced_repositories` is set to true, only the repositories explicitly passed to the scraper will be scraped, ignoring any ignored repositories. Otherwise, the following rules apply:
+
 - For servers that are set to or detected as CVMFS, the scraper will scrape the union of the detected and configurations explicitly stated repositories.
 - For servers that are set to or detected as S3, only the explicitly stated repositories will be scraped (and the scraper will fail if the server type is explicitly set to S3 and no repositories are passed).
 

README.md

@@ -49,6 +49,7 @@
 //!    let scraped_servers = Scraper::new()
 //!       .forced_repositories(repolist)
 //!       .ignored_repositories(ignored_repos)
+//!       .only_scrape_forced_repositories(false) // Only scrape forced repositories if true, overrides ignored_repositories, default false
 //!       .geoapi_servers(DEFAULT_GEOAPI_SERVERS.clone())? // This is the default list
 //!       .with_servers(servers) // Transitions to a WithServer state.
 //!       .validate()? // Transitions to a ValidatedAndReady state, now immutable.
@@ -117,7 +118,7 @@ mod tests {
         let futures = servers.into_iter().map(|server| {
             let repolist = repolist.clone();
             async move {
-                match server.scrape(repolist.clone(), vec![], None).await {
+                match server.scrape(repolist.clone(), vec![], false, None).await {
                     ScrapedServer::Populated(popserver) => {
                         for repo in repolist {
                             assert!(popserver.has_repository(repo));
@@ -143,7 +144,7 @@ mod tests {
 
         let repolist = vec!["software.eessi.io", "dev.eessi.io"];
 
-        match server.scrape(repolist.clone(), vec![], None).await {
+        match server.scrape(repolist.clone(), vec![], false, None).await {
             ScrapedServer::Populated(_) => {
                 panic!("Error, should not have succeeded");
             }
@@ -163,7 +164,7 @@ mod tests {
 
         let repolist = vec!["software.eessi.io", "dev.eessi.io", "riscv.eessi.io"];
         let repoparams: Vec<String> = Vec::new();
-        let servers = server.scrape(repoparams, vec![], None).await;
+        let servers = server.scrape(repoparams, vec![], false, None).await;
         for repo in repolist {
             match servers.clone() {
                 ScrapedServer::Populated(popserver) => {
@@ -186,7 +187,7 @@ mod tests {
 
         let repolist = vec!["software.eessi.io", "dev.eessi.io", "riscv.eessi.io"];
         let popserver = server
-            .scrape(repolist.clone(), vec![], None)
+            .scrape(repolist.clone(), vec![], false, None)
             .await
             .get_populated_server()
             .unwrap();
@@ -203,7 +204,7 @@ mod tests {
         );
 
         let repoparams: Vec<String> = Vec::new();
-        let popserver = server.scrape(repoparams, vec![], None).await;
+        let popserver = server.scrape(repoparams, vec![], false, None).await;
         assert!(popserver.is_ok());
         let popserver = popserver.get_populated_server().unwrap();
         assert_eq!(popserver.backend_type, ServerBackendType::AutoDetect);
@@ -220,7 +221,7 @@ mod tests {
 
         let repolist = vec!["software.eessi.io", "dev.eessi.io", "riscv.eessi.io"];
         let popserver = server
-            .scrape(repolist.clone(), vec![], None)
+            .scrape(repolist.clone(), vec![], false, None)
             .await
             .get_populated_server()
             .unwrap();
@@ -238,7 +239,7 @@ mod tests {
 
         let repolist = vec!["software.eessi.io", "dev.eessi.io", "riscv.eessi.io"];
         let popserver = server
-            .scrape(repolist.clone(), vec![], None)
+            .scrape(repolist.clone(), vec![], false, None)
             .await
             .get_populated_server()
             .unwrap();
@@ -269,7 +270,7 @@ mod tests {
 
         let repolist = vec!["software.eessi.io", "dev.eessi.io", "riscv.eessi.io"];
         let popserver = server
-            .scrape(repolist.clone(), vec![], None)
+            .scrape(repolist.clone(), vec![], false, None)
             .await
             .get_populated_server()
             .unwrap();
@@ -292,7 +293,7 @@ mod tests {
         let repolist = vec!["software.eessi.io", "dev.eessi.io", "riscv.eessi.io"];
         let ignored_repos = vec!["riscv.eessi.io"];
         let popserver = server
-            .scrape(repolist.clone(), ignored_repos.clone(), None)
+            .scrape(repolist.clone(), ignored_repos.clone(), false, None)
             .await
             .get_populated_server()
             .unwrap();
@@ -301,6 +302,26 @@ mod tests {
         assert!(!popserver.has_repository("riscv.eessi.io"));
     }
 
+    #[tokio::test]
+    async fn test_online_cvmfs_server_s1_only_forced_repos() {
+        let server = Server::new(
+            ServerType::Stratum1,
+            ServerBackendType::CVMFS,
+            Hostname::try_from("aws-eu-central-s1.eessi.science").unwrap(),
+        );
+
+        let repolist = vec!["software.eessi.io", "dev.eessi.io"];
+        let popserver = server
+            .scrape(repolist.clone(), vec![], true, None)
+            .await
+            .get_populated_server()
+            .unwrap();
+        assert!(popserver.has_repository("software.eessi.io"));
+        assert!(popserver.has_repository("dev.eessi.io"));
+
+        assert!(popserver.repositories.len() == 2);
+    }
+
     #[tokio::test]
     async fn test_online_scraping_using_builder_interface() {
         let scraper = Scraper::new();

src/lib.rs

@@ -161,16 +161,22 @@ impl Server {
     /// also fetch metadata about the server from the repositories.json and meta.json files, if they
     /// are available.
     ///
-    /// The method takes a list of repositories to scrape (which may be empty unless the backend is S3)
-    /// and a list of repositories to ignore (this may be empty). It also takes an optional list of
-    /// geoapi servers to use when fetching geoapi information.
+    /// ## Arguments
     ///
-    /// The method will return a populated server object if the scrape is successful, or a failed server
-    /// object if the scrape fails.
+    /// - `repositories`: A list of repositories to scrape. This may be empty unless the backend is S3.
+    /// - `ignored_repositories`: A list of repositories to ignore. This may be empty.
+    /// - `only_scrape_forced_repos`: If true, only the repositories provided in the `repositories` argument will be scraped
+    ///    which overrides ignored_repositories. If false, the repositories from repositories.json will be merged with
+    ///    the provided list and then filtered by ignored_repositories.
+    ///
+    /// ## Returns
+    ///
+    /// A ScrapedServer enum containing either a PopulatedServer or a FailedServer.
     pub async fn scrape<R>(
         &self,
         repositories: Vec<R>,
         ignored_repositories: Vec<R>,
+        only_scrape_forced_repos: bool,
         geoapi_servers: Option<Vec<Hostname>>,
     ) -> ScrapedServer
     where
@@ -228,13 +234,16 @@ impl Server {
                         }
                     };
                     backend_detected = ServerBackendType::CVMFS;
-                    all_repos.extend(
-                        repo_json
-                            .repositories_and_replicas()
-                            .into_iter()
-                            .filter(|r| !ignore.contains(&r.name))
-                            .map(|r| r.name),
-                    );
+
+                    if !only_scrape_forced_repos {
+                        all_repos.extend(
+                            repo_json
+                                .repositories_and_replicas()
+                                .into_iter()
+                                .filter(|r| !ignore.contains(&r.name))
+                                .map(|r| r.name),
+                        );
+                    };
                 }
                 Err(error) => match error {
                     ScrapeError::FetchError(_) => {
@@ -274,13 +283,15 @@ impl Server {
                         return ScrapedServer::Failed(self.to_failed_server(error));
                     }
                 }
-                all_repos.extend(
-                    repo_json
-                        .repositories_and_replicas()
-                        .into_iter()
-                        .filter(|r| !ignore.contains(&r.name))
-                        .map(|r| r.name),
-                );
+                if !only_scrape_forced_repos {
+                    all_repos.extend(
+                        repo_json
+                            .repositories_and_replicas()
+                            .into_iter()
+                            .filter(|r| !ignore.contains(&r.name))
+                            .map(|r| r.name),
+                    )
+                };
             }
         }