feature(client-lib,client-cli): add arg --allow-missing in cardano-db verify command

Author: Clément Turmel

mithril-client-cli/src/commands/cardano_db/download/v2.rs

@@ -15,7 +15,10 @@ use mithril_client::{
 use crate::{
     CommandContext,
     commands::{
-        cardano_db::{download::DB_DIRECTORY_NAME, shared_steps},
+        cardano_db::{
+            download::DB_DIRECTORY_NAME,
+            shared_steps::{self, ComputeCardanoDatabaseMessageOptions},
+        },
         client_builder,
     },
     utils::{
@@ -139,13 +142,17 @@ impl PreparedCardanoDbV2Download {
             )
         })?;
 
+        let options = ComputeCardanoDatabaseMessageOptions {
+            db_dir: restoration_options.db_dir.clone(),
+            immutable_file_range: restoration_options.immutable_file_range,
+            allow_missing: false,
+        };
         let message = shared_steps::compute_cardano_db_snapshot_message(
             5,
             &progress_printer,
             &certificate,
             &cardano_db_message,
-            &restoration_options.immutable_file_range,
-            &restoration_options.db_dir,
+            &options,
             &verified_digests,
         )
         .await?;

mithril-client-cli/src/commands/cardano_db/shared_steps.rs

@@ -1,7 +1,7 @@
 use anyhow::{Context, anyhow};
 use chrono::Utc;
 use slog::{Logger, debug, warn};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use mithril_client::{
     CardanoDatabaseSnapshot, Client, MessageBuilder, MithrilCertificate, MithrilResult,
@@ -11,6 +11,12 @@ use mithril_client::{
 
 use crate::utils::{CardanoDbUtils, ProgressPrinter};
 
+pub struct ComputeCardanoDatabaseMessageOptions {
+    pub db_dir: PathBuf,
+    pub immutable_file_range: ImmutableFileRange,
+    pub allow_missing: bool,
+}
+
 pub async fn fetch_certificate_and_verifying_chain(
     step_number: u16,
     progress_printer: &ProgressPrinter,
@@ -69,8 +75,7 @@ pub async fn compute_cardano_db_snapshot_message(
     progress_printer: &ProgressPrinter,
     certificate: &MithrilCertificate,
     cardano_database_snapshot: &CardanoDatabaseSnapshot,
-    immutable_file_range: &ImmutableFileRange,
-    database_dir: &Path,
+    options: &ComputeCardanoDatabaseMessageOptions,
     verified_digest: &VerifiedDigests,
 ) -> MithrilResult<ProtocolMessage> {
     progress_printer.report_step(step_number, "Computing the cardano db snapshot message")?;
@@ -79,8 +84,9 @@ pub async fn compute_cardano_db_snapshot_message(
         MessageBuilder::new().compute_cardano_database_message(
             certificate,
             cardano_database_snapshot,
-            immutable_file_range,
-            database_dir,
+            &options.immutable_file_range,
+            options.allow_missing,
+            &options.db_dir,
             verified_digest,
         ),
     )

mithril-client-cli/src/commands/cardano_db/verify.rs

@@ -15,7 +15,10 @@ use mithril_client::{
 use crate::{
     CommandContext,
     commands::{
-        cardano_db::{CardanoDbCommandsBackend, shared_steps},
+        cardano_db::{
+            CardanoDbCommandsBackend,
+            shared_steps::{self, ComputeCardanoDatabaseMessageOptions},
+        },
         client_builder,
     },
     configuration::{ConfigError, ConfigSource},
@@ -53,6 +56,10 @@ pub struct CardanoDbVerifyCommand {
     /// If not set, the verify will continue until the last certified immutable file.
     #[clap(long)]
     end: Option<ImmutableFileNumber>,
+
+    /// If set, the verification will not fail if some immutable files are missing.
+    #[clap(long)]
+    allow_missing: bool,
 }
 
 impl CardanoDbVerifyCommand {
@@ -134,13 +141,17 @@ impl CardanoDbVerifyCommand {
         )
         .await?;
 
+        let options = ComputeCardanoDatabaseMessageOptions {
+            db_dir: db_dir.to_path_buf(),
+            immutable_file_range,
+            allow_missing: self.allow_missing,
+        };
         let message = shared_steps::compute_cardano_db_snapshot_message(
             3,
             &progress_printer,
             &certificate,
             &cardano_db_message,
-            &immutable_file_range,
-            db_dir,
+            &options,
             &verified_digests,
         )
         .await;

mithril-client/src/message.rs

@@ -230,15 +230,19 @@ impl MessageBuilder {
             certificate: &CertificateMessage,
             cardano_database_snapshot: &CardanoDatabaseSnapshotMessage,
             immutable_file_range: &ImmutableFileRange,
+            allow_missing: bool,
             database_dir: &Path,
             verified_digests: &VerifiedDigests,
         ) -> Result<ProtocolMessage, ComputeCardanoDatabaseMessageError> {
             let network = certificate.metadata.network.clone();
             let immutable_file_number_range = immutable_file_range
                 .to_range_inclusive(cardano_database_snapshot.beacon.immutable_file_number)
                 .map_err(ComputeCardanoDatabaseMessageError::ImmutableFilesRange)?;
-            let missing_immutable_files =
-                Self::list_missing_immutable_files(database_dir, &immutable_file_number_range);
+            let missing_immutable_files = if allow_missing {
+                vec![]
+            } else {
+                Self::list_missing_immutable_files(database_dir, &immutable_file_number_range)
+            };
             let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone());
             let computed_digest_entries = immutable_digester
                 .compute_digests_for_range(database_dir, &immutable_file_number_range)
@@ -533,6 +537,7 @@ mod tests {
                     &certificate,
                     &CardanoDatabaseSnapshotMessage::dummy(),
                     &immutable_file_range_to_prove,
+                    false,
                     &database_dir,
                     &verified_digests,
                 )
@@ -543,15 +548,16 @@ mod tests {
         }
 
         #[tokio::test]
-        async fn compute_cardano_database_message_should_fail_if_immutable_is_missing() {
+        async fn compute_cardano_database_message_should_fail_if_immutable_is_missing_and_allow_missing_not_set()
+         {
             let beacon = CardanoDbBeacon {
                 epoch: Epoch(123),
                 immutable_file_number: 10,
             };
             let immutable_file_range = 1..=15;
             let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4);
             let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests(
-                "compute_cardano_database_message_should_fail_if_immutable_is_missing",
+                "compute_cardano_database_message_should_fail_if_immutable_is_missing_and_allow_missing_not_set",
                 &beacon,
                 &immutable_file_range,
             )
@@ -560,11 +566,13 @@ mod tests {
             let files_to_remove = vec!["00003.chunk", "00004.primary"];
             remove_immutable_files(&database_dir, &files_to_remove);
 
+            let allow_missing = false;
             let error = MessageBuilder::new()
                 .compute_cardano_database_message(
                     &certificate,
                     &CardanoDatabaseSnapshotMessage::dummy(),
                     &immutable_file_range_to_prove,
+                    allow_missing,
                     &database_dir,
                     &verified_digests,
                 )
@@ -588,6 +596,41 @@ mod tests {
             );
         }
 
+        #[tokio::test]
+        async fn compute_cardano_database_message_should_success_if_immutable_is_missing_and_allow_missing_is_set()
+         {
+            let beacon = CardanoDbBeacon {
+                epoch: Epoch(123),
+                immutable_file_number: 10,
+            };
+            let immutable_file_range = 1..=15;
+            let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4);
+            let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests(
+                "compute_cardano_database_message_should_success_if_immutable_is_missing_and_allow_missing_is_set",
+                &beacon,
+                &immutable_file_range,
+            )
+            .await;
+
+            let files_to_remove = vec!["00003.chunk", "00004.primary"];
+            remove_immutable_files(&database_dir, &files_to_remove);
+
+            let allow_missing = true;
+            MessageBuilder::new()
+                .compute_cardano_database_message(
+                    &certificate,
+                    &CardanoDatabaseSnapshotMessage::dummy(),
+                    &immutable_file_range_to_prove,
+                    allow_missing,
+                    &database_dir,
+                    &verified_digests,
+                )
+                .await
+                .expect(
+                    "compute_cardano_database_message should succeed if a immutable is missing but 'allow_missing' is set",
+                );
+        }
+
         #[tokio::test]
         async fn compute_cardano_database_message_should_fail_if_immutable_is_tampered() {
             let beacon = CardanoDbBeacon {
@@ -614,6 +657,7 @@ mod tests {
                     &certificate,
                     &CardanoDatabaseSnapshotMessage::dummy(),
                     &immutable_file_range_to_prove,
+                    false,
                     &database_dir,
                     &verified_digests,
                 )
@@ -665,6 +709,7 @@ mod tests {
                     &certificate,
                     &CardanoDatabaseSnapshotMessage::dummy(),
                     &immutable_file_range_to_prove,
+                    false,
                     &database_dir,
                     &verified_digests,
                 )

mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs

@@ -144,6 +144,7 @@ async fn cardano_db_snapshot_list_get_download_verify() {
             &certificate,
             &cardano_db_snapshot,
             &immutable_file_range,
+            false,
             &unpacked_dir,
             &verified_digests,
         )