From 6cff1d68416276c12fdd3524240c423725a1a00b Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Jun 2025 14:31:23 +0200 Subject: [PATCH 01/38] chore: scaffold 'internal/mithril-dmq-node' crate --- .github/workflows/ci.yml | 1 + Cargo.lock | 99 ++++++++++++++++++++++++---- Cargo.toml | 1 + Makefile | 1 + README.md | 2 + internal/mithril-dmq-node/Cargo.toml | 18 +++++ internal/mithril-dmq-node/Makefile | 19 ++++++ internal/mithril-dmq-node/README.md | 5 ++ internal/mithril-dmq-node/src/lib.rs | 2 + 9 files changed, 134 insertions(+), 14 deletions(-) create mode 100644 internal/mithril-dmq-node/Cargo.toml create mode 100644 internal/mithril-dmq-node/Makefile create mode 100644 internal/mithril-dmq-node/README.md create mode 100644 internal/mithril-dmq-node/src/lib.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acfcc45ea6b..22f2422a952 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -805,6 +805,7 @@ jobs: # the same name (we only want to document those anyway) cargo doc --no-deps --lib -p mithril-stm -p mithril-common \ -p mithril-cardano-node-chain -p mithril-cardano-node-internal-database \ + -p mithril-dmq-node \ -p mithril-build-script -p mithril-cli-helper -p mithril-doc -p mithril-doc-derive \ -p mithril-era -p mithril-metric -p mithril-persistence -p mithril-resource-pool \ -p mithril-ticker -p mithril-signed-entity-lock -p mithril-signed-entity-preloader \ diff --git a/Cargo.lock b/Cargo.lock index 09f2d150883..ac8d3253b99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3837,7 +3837,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0452a60c1863c1f50b5f77cd295e8d2786849f35883f0b9e18e7e6e1b5691b0" dependencies = [ "half", - "minicbor-derive", + "minicbor-derive 0.15.3", +] + +[[package]] +name = "minicbor" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a309f581ade7597820083bc275075c4c6986e57e53f8d26f88507cfefc8c987" +dependencies = [ + "half", + "minicbor-derive 0.16.2", ] [[package]] @@ -3851,6 +3861,17 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "minicbor-derive" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9882ef5c56df184b8ffc107fc6c61e33ee3a654b021961d790a78571bb9d67a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "minicov" version = "0.3.7" @@ -3999,9 +4020,9 @@ dependencies = [ "mockall", "nom 8.0.0", "pallas-addresses", - "pallas-codec", - "pallas-crypto", - "pallas-network", + "pallas-codec 0.32.0", + "pallas-crypto 0.32.0", + "pallas-network 0.32.0", "pallas-primitives", "pallas-traverse", "rand_core 0.6.4", @@ -4178,6 +4199,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "mithril-dmq-node" +version = "0.1.0" +dependencies = [ + "blake2 0.10.6", + "mithril-common", + "pallas-network 1.0.0-alpha.2", +] + [[package]] name = "mithril-doc" version = "0.1.24" @@ -4930,8 +4960,8 @@ dependencies = [ "crc", "cryptoxide", "hex", - "pallas-codec", - "pallas-crypto", + "pallas-codec 0.32.0", + "pallas-crypto 0.32.0", "thiserror 1.0.69", ] @@ -4942,7 +4972,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e344b3e39ca3bd79bb7547b65b980869c3c377a00c48ece70430f4611c32a18b" dependencies = [ "hex", - "minicbor", + "minicbor 0.25.1", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "pallas-codec" +version = "1.0.0-alpha.2" +source = "git+https://github.com/txpipe/pallas.git?branch=main#a97bd93cdc55fa2b061a6ad5fd572f5528a912b8" +dependencies = [ + "hex", + "minicbor 0.26.5", "serde", "thiserror 1.0.69", ] @@ -4955,13 +4996,26 @@ checksum = "59c89ea16190a87a1d8bd36923093740a2b659ed6129f4636329319a70cc4db3" dependencies = [ "cryptoxide", "hex", - "pallas-codec", + "pallas-codec 0.32.0", "rand_core 0.6.4", "serde", "thiserror 1.0.69", "zeroize", ] +[[package]] +name = "pallas-crypto" +version = "1.0.0-alpha.2" +source = "git+https://github.com/txpipe/pallas.git?branch=main#a97bd93cdc55fa2b061a6ad5fd572f5528a912b8" +dependencies = [ + "cryptoxide", + "hex", + "pallas-codec 1.0.0-alpha.2", + "rand_core 0.6.4", + "serde", + "thiserror 1.0.69", +] + [[package]] name = "pallas-network" version = "0.32.1" @@ -4971,8 +5025,25 @@ dependencies = [ "byteorder", "hex", "itertools 0.13.0", - "pallas-codec", - "pallas-crypto", + "pallas-codec 0.32.0", + "pallas-crypto 0.32.0", + "rand 0.8.5", + "socket2", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "pallas-network" +version = "1.0.0-alpha.2" +source = "git+https://github.com/txpipe/pallas.git?branch=main#a97bd93cdc55fa2b061a6ad5fd572f5528a912b8" +dependencies = [ + "byteorder", + "hex", + "itertools 0.13.0", + "pallas-codec 1.0.0-alpha.2", + "pallas-crypto 1.0.0-alpha.2", "rand 0.8.5", "socket2", "thiserror 1.0.69", @@ -4990,8 +5061,8 @@ dependencies = [ "bech32 0.9.1", "hex", "log", - "pallas-codec", - "pallas-crypto", + "pallas-codec 0.32.0", + "pallas-crypto 0.32.0", "serde", "serde_json", ] @@ -5005,8 +5076,8 @@ dependencies = [ "hex", "itertools 0.13.0", "pallas-addresses", - "pallas-codec", - "pallas-crypto", + "pallas-codec 0.32.0", + "pallas-crypto 0.32.0", "pallas-primitives", "paste", "serde", diff --git a/Cargo.toml b/Cargo.toml index fe906161c2a..ebc25ecb852 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "internal/cardano-node/mithril-cardano-node-internal-database", "internal/mithril-build-script", "internal/mithril-cli-helper", + "internal/mithril-dmq-node", "internal/mithril-doc", "internal/mithril-doc-derive", "internal/mithril-era", diff --git a/Makefile b/Makefile index 05a1a270dd5..c6451c4fd80 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ COMPONENTS = mithril-aggregator mithril-client mithril-client-cli mithril-client-wasm \ mithril-common mithril-relay mithril-signer mithril-stm \ internal/mithril-build-script internal/mithril-cli-helper internal/mithril-doc \ + internal/mithril-dmq-node \ internal/mithril-doc-derive internal/mithril-era internal/mithril-metric internal/mithril-persistence \ internal/mithril-resource-pool internal/mithril-ticker \ internal/cardano-node/mithril-cardano-node-chain internal/cardano-node/mithril-cardano-node-internal-database \ diff --git a/README.md b/README.md index 95b9b93297e..2886c6369e5 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,8 @@ This repository consists of the following parts: - [**Mithril cli helper**](./internal/mithril-cli-helper): **CLI** tools for **Mithril** binaries. + - [**Mithril DMQ node**](./internal/mithril-dmq-node): mechanisms to publish and consume messages of a **Decentralized Message Queue network** through a DMQ node, used by Mithril network nodes. + - [**Mithril doc**](./internal/mithril-doc): an API that generates Markdown documentation for crate command line arguments. - [**Mithril doc derive**](./internal/mithril-doc-derive): a macro implementation used by **Mithril doc**. diff --git a/internal/mithril-dmq-node/Cargo.toml b/internal/mithril-dmq-node/Cargo.toml new file mode 100644 index 00000000000..53f980644f3 --- /dev/null +++ b/internal/mithril-dmq-node/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "mithril-dmq-node" +version = "0.1.0" +authors.workspace = true +documentation.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +include = ["**/*.rs", "Cargo.toml", "README.md", ".gitignore"] + +[lib] +crate-type = ["lib", "cdylib", "staticlib"] + +[dependencies] +blake2 = "0.10.6" +mithril-common = { path = "../../mithril-common" } +pallas-network = { git = "https://github.com/txpipe/pallas.git", branch = "main", optional = true } \ No newline at end of file diff --git a/internal/mithril-dmq-node/Makefile b/internal/mithril-dmq-node/Makefile new file mode 100644 index 00000000000..e503264d97d --- /dev/null +++ b/internal/mithril-dmq-node/Makefile @@ -0,0 +1,19 @@ +.PHONY: all build test check doc + +CARGO = cargo + +all: test build + +build: + ${CARGO} build --release + +test: + ${CARGO} test + +check: + ${CARGO} check --release --all-features --all-targets + ${CARGO} clippy --release --all-features --all-targets + ${CARGO} fmt --check + +doc: + ${CARGO} doc --no-deps --open \ No newline at end of file diff --git a/internal/mithril-dmq-node/README.md b/internal/mithril-dmq-node/README.md new file mode 100644 index 00000000000..fdeb4df373e --- /dev/null +++ b/internal/mithril-dmq-node/README.md @@ -0,0 +1,5 @@ +# Mithril-dmq-node + +This crate provides mechanisms to publish and consume messages of a **Decentralized Message Queue network** through a DMQ node. + +The full protocol is defined in the [CIP-0137](https://github.com/cardano-foundation/CIPs/tree/master/CIP-0137#local-message-notification-mini-protocol). diff --git a/internal/mithril-dmq-node/src/lib.rs b/internal/mithril-dmq-node/src/lib.rs new file mode 100644 index 00000000000..cc96324c6d7 --- /dev/null +++ b/internal/mithril-dmq-node/src/lib.rs @@ -0,0 +1,2 @@ +#![warn(missing_docs)] +//! This crate provides mechanisms to publish and consume messages of a Decentralized Message Queue network through a DMQ node. From dff43f589482764a9367c5f549c7299eaf0d6787 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Jun 2025 15:59:34 +0200 Subject: [PATCH 02/38] feat(dmq): add 'DmqMessageBuilder' --- Cargo.lock | 6 + internal/mithril-dmq-node/Cargo.toml | 12 +- internal/mithril-dmq-node/src/lib.rs | 29 +++++ internal/mithril-dmq-node/src/message.rs | 137 +++++++++++++++++++++++ 4 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 internal/mithril-dmq-node/src/message.rs diff --git a/Cargo.lock b/Cargo.lock index ac8d3253b99..96967db0ea9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4203,9 +4203,15 @@ dependencies = [ name = "mithril-dmq-node" version = "0.1.0" dependencies = [ + "anyhow", "blake2 0.10.6", + "mithril-cardano-node-chain", "mithril-common", "pallas-network 1.0.0-alpha.2", + "slog", + "slog-async", + "slog-term", + "tokio", ] [[package]] diff --git a/internal/mithril-dmq-node/Cargo.toml b/internal/mithril-dmq-node/Cargo.toml index 53f980644f3..af6e6c9726c 100644 --- a/internal/mithril-dmq-node/Cargo.toml +++ b/internal/mithril-dmq-node/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "mithril-dmq-node" +description = "Mechanisms to publish and consume messages of a 'Decentralized Message Queue network' through a DMQ node" version = "0.1.0" authors.workspace = true documentation.workspace = true @@ -13,6 +14,15 @@ include = ["**/*.rs", "Cargo.toml", "README.md", ".gitignore"] crate-type = ["lib", "cdylib", "staticlib"] [dependencies] +anyhow = { workspace = true } blake2 = "0.10.6" +mithril-cardano-node-chain = { path = "../cardano-node/mithril-cardano-node-chain" } mithril-common = { path = "../../mithril-common" } -pallas-network = { git = "https://github.com/txpipe/pallas.git", branch = "main", optional = true } \ No newline at end of file +pallas-network = { git = "https://github.com/txpipe/pallas.git", branch = "main" } +slog = { workspace = true } +tokio = { workspace = true } + +[dev-dependencies] +mithril-common = { path = "../../mithril-common", features = ["test_tools"] } +slog-async = { workspace = true } +slog-term = { workspace = true } \ No newline at end of file diff --git a/internal/mithril-dmq-node/src/lib.rs b/internal/mithril-dmq-node/src/lib.rs index cc96324c6d7..a7d08725923 100644 --- a/internal/mithril-dmq-node/src/lib.rs +++ b/internal/mithril-dmq-node/src/lib.rs @@ -1,2 +1,31 @@ #![warn(missing_docs)] //! This crate provides mechanisms to publish and consume messages of a Decentralized Message Queue network through a DMQ node. + +mod message; + +pub use message::DmqMessageBuilder; + +#[cfg(test)] +pub(crate) mod test_tools { + use std::io; + use std::sync::Arc; + + use slog::{Drain, Logger}; + use slog_async::Async; + use slog_term::{CompactFormat, PlainDecorator}; + + pub struct TestLogger; + + impl TestLogger { + fn from_writer(writer: W) -> Logger { + let decorator = PlainDecorator::new(writer); + let drain = CompactFormat::new(decorator).build().fuse(); + let drain = Async::new(drain).build().fuse(); + Logger::root(Arc::new(drain), slog::o!()) + } + + pub fn stdout() -> Logger { + Self::from_writer(slog_term::TestStdoutWriter) + } + } +} diff --git a/internal/mithril-dmq-node/src/message.rs b/internal/mithril-dmq-node/src/message.rs new file mode 100644 index 00000000000..8d94dfba143 --- /dev/null +++ b/internal/mithril-dmq-node/src/message.rs @@ -0,0 +1,137 @@ +use std::sync::Arc; + +use anyhow::{anyhow, Context}; +use blake2::{digest::consts::U64, Blake2b, Digest}; +use pallas_network::miniprotocols::localmsgsubmission::DmqMsg; + +use mithril_cardano_node_chain::chain_observer::ChainObserver; +use mithril_common::StdResult; + +/// The TTL (Time To Live) for DMQ messages in blocks. +const DMQ_MESSAGE_TTL_IN_BLOCKS: u16 = 100; + +/// A builder for creating DMQ messages. +pub struct DmqMessageBuilder { + chain_observer: Arc, + ttl_blocks: u16, +} + +impl DmqMessageBuilder { + /// Creates a new instance of `DmqMessageBuilder`. + pub fn new(chain_observer: Arc, ttl_blocks: u16) -> Self { + Self { + chain_observer, + ttl_blocks, + } + } + + /// Creates a new instance of `DmqMessageBuilder` with default TTL. + pub fn new_with_default_ttl(chain_observer: Arc) -> Self { + Self { + chain_observer, + ttl_blocks: DMQ_MESSAGE_TTL_IN_BLOCKS, + } + } + + /// Builds a DMQ message from the provided message bytes. + pub async fn build(&self, message_bytes: &[u8]) -> StdResult { + fn compute_msg_id(dmq_message: &DmqMsg) -> Vec { + let mut hasher = Blake2b::::new(); + hasher.update(&dmq_message.msg_body); + hasher.update(dmq_message.block_number.to_be_bytes()); + hasher.update(dmq_message.ttl.to_be_bytes()); + hasher.update(&dmq_message.kes_signature); + hasher.update(&dmq_message.operational_certificate); + + hasher.finalize().to_vec() + } + + let block_number = self + .chain_observer + .get_current_chain_point() + .await + .with_context(|| "Failed to get current chain point while building DMQ message")? + .ok_or(anyhow!( + "No current chain point available while building DMQ message" + ))? + .block_number; + let block_number = (*block_number) + .try_into() + .map_err(|_| anyhow!("Failed to convert block number to u32"))?; + let kes_signature = vec![]; // TO DO: create a KES signature + let operational_certificate = vec![]; // TO DO: create an operational certificate + let mut dmq_message = DmqMsg { + msg_id: vec![], + msg_body: message_bytes.to_vec(), + block_number, + ttl: self.ttl_blocks, + kes_signature, + operational_certificate, + }; + dmq_message.msg_id = compute_msg_id(&dmq_message); + + Ok(dmq_message) + } +} + +#[cfg(test)] +mod tests { + use mithril_cardano_node_chain::test::double::FakeChainObserver; + use mithril_common::{ + crypto_helper::TryToBytes, + entities::{BlockNumber, ChainPoint, TimePoint}, + }; + + use super::*; + + mod test_utils { + use super::*; + + pub(super) struct TestMessage { + pub(super) content: Vec, + } + + impl TryToBytes for TestMessage { + fn to_bytes_vec(&self) -> StdResult> { + Ok(self.content.clone()) + } + } + } + + #[tokio::test] + async fn test_build_dmq_message() { + let chain_observer = Arc::new(FakeChainObserver::new(Some(TimePoint { + chain_point: ChainPoint { + block_number: BlockNumber(123), + ..ChainPoint::dummy() + }, + ..TimePoint::dummy() + }))); + let builder = DmqMessageBuilder::new(chain_observer, 100); + let message = test_utils::TestMessage { + content: b"test".to_vec(), + }; + + let dmq_message = builder + .build(&message.to_bytes_vec().unwrap()) + .await + .unwrap(); + + assert_eq!( + DmqMsg { + msg_id: vec![ + 26, 113, 171, 177, 174, 241, 244, 241, 209, 92, 210, 7, 119, 105, 94, 133, 93, + 62, 82, 95, 91, 221, 146, 174, 201, 190, 140, 1, 217, 240, 228, 203, 14, 50, + 104, 59, 252, 216, 26, 84, 231, 142, 163, 140, 11, 95, 17, 234, 242, 39, 230, + 160, 194, 219, 128, 42, 53, 125, 218, 48, 209, 3, 210, 154 + ], + msg_body: vec![116, 101, 115, 116], + block_number: 123, + ttl: 100, + kes_signature: vec![], + operational_certificate: vec![], + }, + dmq_message + ); + } +} From 90991b9ed3a8e502917f15ba03481f2c399a50b0 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Jun 2025 16:00:18 +0200 Subject: [PATCH 03/38] feat(dmq): add 'test' module and test payload --- internal/mithril-dmq-node/src/lib.rs | 1 + internal/mithril-dmq-node/src/test/mod.rs | 5 ++ internal/mithril-dmq-node/src/test/payload.rs | 50 +++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 internal/mithril-dmq-node/src/test/mod.rs create mode 100644 internal/mithril-dmq-node/src/test/payload.rs diff --git a/internal/mithril-dmq-node/src/lib.rs b/internal/mithril-dmq-node/src/lib.rs index a7d08725923..e7ec4091948 100644 --- a/internal/mithril-dmq-node/src/lib.rs +++ b/internal/mithril-dmq-node/src/lib.rs @@ -2,6 +2,7 @@ //! This crate provides mechanisms to publish and consume messages of a Decentralized Message Queue network through a DMQ node. mod message; +pub mod test; pub use message::DmqMessageBuilder; diff --git a/internal/mithril-dmq-node/src/test/mod.rs b/internal/mithril-dmq-node/src/test/mod.rs new file mode 100644 index 00000000000..0363df24048 --- /dev/null +++ b/internal/mithril-dmq-node/src/test/mod.rs @@ -0,0 +1,5 @@ +//! Test utilities. +//! +//! ⚠ Do not use in production code ⚠ + +pub(crate) mod payload; diff --git a/internal/mithril-dmq-node/src/test/payload.rs b/internal/mithril-dmq-node/src/test/payload.rs new file mode 100644 index 00000000000..4dbb829afac --- /dev/null +++ b/internal/mithril-dmq-node/src/test/payload.rs @@ -0,0 +1,50 @@ +use std::fmt::Debug; + +use mithril_common::{ + crypto_helper::{TryFromBytes, TryToBytes}, + StdResult, +}; + +/// A test message payload for the DMQ. +#[derive(PartialEq, Eq)] +pub struct DmqMessageTestPayload { + message: Vec, +} + +impl DmqMessageTestPayload { + /// Creates a new `DmqMessageTestPayload` with the given bytes. + pub fn new(bytes: &[u8]) -> Self { + Self { + message: bytes.to_vec(), + } + } + + /// Creates a dummy `DmqMessageTestPayload` with a predefined message. + pub fn dummy() -> Self { + Self { + message: b"dummy message".to_vec(), + } + } +} + +impl Debug for DmqMessageTestPayload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DmqMessageTestPayload") + .field("message", &self.message) + .finish() + } +} + +impl TryToBytes for DmqMessageTestPayload { + fn to_bytes_vec(&self) -> StdResult> { + Ok(self.message.clone()) + } +} + +impl TryFromBytes for DmqMessageTestPayload { + fn try_from_bytes(bytes: &[u8]) -> StdResult { + Ok(Self { + message: bytes.to_vec(), + }) + } +} From 32ee7fec6f50ecb6df77255f6d88ba3facf535e9 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Jun 2025 16:03:32 +0200 Subject: [PATCH 04/38] feat(dmq): add 'DmqPublisherPallas' implementation --- internal/mithril-dmq-node/src/lib.rs | 2 + internal/mithril-dmq-node/src/publisher.rs | 185 +++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 internal/mithril-dmq-node/src/publisher.rs diff --git a/internal/mithril-dmq-node/src/lib.rs b/internal/mithril-dmq-node/src/lib.rs index e7ec4091948..6e34b58d5ba 100644 --- a/internal/mithril-dmq-node/src/lib.rs +++ b/internal/mithril-dmq-node/src/lib.rs @@ -2,9 +2,11 @@ //! This crate provides mechanisms to publish and consume messages of a Decentralized Message Queue network through a DMQ node. mod message; +mod publisher; pub mod test; pub use message::DmqMessageBuilder; +pub use publisher::DmqPublisherPallas; #[cfg(test)] pub(crate) mod test_tools { diff --git a/internal/mithril-dmq-node/src/publisher.rs b/internal/mithril-dmq-node/src/publisher.rs new file mode 100644 index 00000000000..f3466e00cf0 --- /dev/null +++ b/internal/mithril-dmq-node/src/publisher.rs @@ -0,0 +1,185 @@ +use std::{fmt::Debug, marker::PhantomData, path::PathBuf}; + +use anyhow::{anyhow, Context}; +use pallas_network::{facades::DmqClient, miniprotocols::localtxsubmission::Response}; +use slog::{debug, error, Logger}; + +use mithril_common::{ + crypto_helper::TryToBytes, logging::LoggerExtensions, CardanoNetwork, StdResult, +}; + +use crate::DmqMessageBuilder; + +/// A DMQ publisher implementation. +/// This implementation is built upon the n2c mini-protocols DMQ implementation in Pallas. +pub struct DmqPublisherPallas { + socket: PathBuf, + network: CardanoNetwork, + dmq_message_builder: DmqMessageBuilder, + logger: Logger, + phantom: PhantomData, +} + +impl DmqPublisherPallas { + /// Creates a new instance of [DmqPublisherPallas]. + pub fn new( + socket: PathBuf, + network: CardanoNetwork, + dmq_message_builder: DmqMessageBuilder, + logger: Logger, + ) -> Self { + Self { + socket, + network, + dmq_message_builder, + logger: logger.new_with_component_name::(), + phantom: PhantomData, + } + } + + /// Creates and returns a new `DmqClient` connected to the specified socket. + async fn new_client(&self) -> StdResult { + let magic = self.network.code(); + DmqClient::connect(&self.socket, magic) + .await + .map_err(|err| anyhow!(err)) + .with_context(|| "PallasChainReader failed to create a new client") + } + + /// Publishes a message to the DMQ node. + pub async fn publish_message(&self, message: M) -> StdResult<()> { + debug!( + self.logger, + "Publish message to DMQ"; + "message" => ?message + ); + let mut client = self.new_client().await?; + let message_bytes = &message.to_bytes_vec()?; + let dmq_message = self + .dmq_message_builder + .build(message_bytes) + .await + .with_context(|| "Failed to build DMQ message")?; + client + .msg_submission() + .send_submit_tx(dmq_message) + .await + .map_err(|err| anyhow!("Failed to submit DMQ message: {}", err))?; + let response = client.msg_submission().recv_submit_tx_response().await?; + if let Err(e) = client.msg_submission().terminate_gracefully().await { + error!(self.logger, "Failed to send Done"; "error" => ?e); + } + + if response != Response::Accepted { + return Err(anyhow!("Failed to publish DMQ message: {:?}", response)); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + + use std::{fs, sync::Arc}; + + use pallas_network::miniprotocols::{ + localmsgsubmission::DmqMsgValidationError, localtxsubmission, + }; + use tokio::{net::UnixListener, task::JoinHandle}; + + use mithril_cardano_node_chain::test::double::FakeChainObserver; + use mithril_common::{current_function, test_utils::TempDir}; + + use crate::{test::payload::DmqMessageTestPayload, test_tools::TestLogger}; + + use super::*; + + fn create_temp_dir(folder_name: &str) -> PathBuf { + TempDir::create_with_short_path("dmq_publisher", folder_name) + } + + fn setup_dmq_server(socket_path: PathBuf, reply_success: bool) -> JoinHandle<()> { + tokio::spawn({ + async move { + // server setup + if socket_path.exists() { + fs::remove_file(socket_path.clone()).unwrap(); + } + let listener = UnixListener::bind(socket_path).unwrap(); + let mut server = pallas_network::facades::DmqServer::accept(&listener, 0) + .await + .unwrap(); + + // init local msg submission server + let server_msg = server.msg_submission(); + + // server waits for request from client and replies to it + let request = server_msg.recv_next_request().await.unwrap(); + if let localtxsubmission::Request::Submit(_) = &request { + } else { + panic!("Expected a submit request, but got: {:?}", request); + } + let response = if reply_success { + localtxsubmission::Response::Accepted + } else { + localtxsubmission::Response::Rejected(DmqMsgValidationError( + "fake error".to_string(), + )) + }; + server_msg.send_submit_tx_response(response).await.unwrap(); + + // server receives done from client + let request = server_msg.recv_next_request().await.unwrap(); + assert_eq!(localtxsubmission::Request::Done, request); + } + }) + } + + #[tokio::test] + async fn pallas_dmq_signature_publisher_success() { + let socket_path = create_temp_dir(current_function!()).join("node.socket"); + let reply_success = true; + let server = setup_dmq_server(socket_path.clone(), reply_success); + let client = tokio::spawn(async move { + let publisher = DmqPublisherPallas::new( + socket_path, + CardanoNetwork::TestNet(0), + DmqMessageBuilder::new(Arc::new(FakeChainObserver::default()), 100), + TestLogger::stdout(), + ); + + publisher + .publish_message(DmqMessageTestPayload::dummy()) + .await + }); + + let (_, res) = tokio::join!(server, client); + + res.unwrap().unwrap(); + } + + #[tokio::test] + async fn pallas_dmq_signature_publisher_fails() { + let socket_path = create_temp_dir(current_function!()).join("node.socket"); + let reply_success = false; + let server = setup_dmq_server(socket_path.clone(), reply_success); + let client = tokio::spawn(async move { + let publisher = DmqPublisherPallas::new( + socket_path, + CardanoNetwork::TestNet(0), + DmqMessageBuilder::new(Arc::new(FakeChainObserver::default()), 100), + TestLogger::stdout(), + ); + + publisher + .publish_message(DmqMessageTestPayload::dummy()) + .await + }); + + let (_, res) = tokio::join!(server, client); + + res.unwrap() + .expect_err("Publishing DMQ message should fail"); + } +} From 0ca120fb3cc52c42a9f9ada670aa1b5a8ddd62b3 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Jun 2025 16:03:47 +0200 Subject: [PATCH 05/38] feat(dmq): add 'DmqConsumerPallas' implementation --- internal/mithril-dmq-node/src/consumer.rs | 226 ++++++++++++++++++++++ internal/mithril-dmq-node/src/lib.rs | 2 + 2 files changed, 228 insertions(+) create mode 100644 internal/mithril-dmq-node/src/consumer.rs diff --git a/internal/mithril-dmq-node/src/consumer.rs b/internal/mithril-dmq-node/src/consumer.rs new file mode 100644 index 00000000000..e1d3ddc48bd --- /dev/null +++ b/internal/mithril-dmq-node/src/consumer.rs @@ -0,0 +1,226 @@ +use std::{fmt::Debug, marker::PhantomData, path::PathBuf}; + +use anyhow::{anyhow, Context}; +use pallas_network::facades::DmqClient; +use slog::{debug, error, Logger}; + +use mithril_common::{ + crypto_helper::{OpCert, TryFromBytes}, + entities::PartyId, + logging::LoggerExtensions, + CardanoNetwork, StdResult, +}; + +/// A DMQ consumer implementation. +/// This implementation is built upon the n2c mini-protocols DMQ implementation in Pallas. +pub struct DmqConsumerPallas { + socket: PathBuf, + network: CardanoNetwork, + logger: Logger, + phantom: PhantomData, +} + +impl DmqConsumerPallas { + /// Creates a new `DmqConsumerPallas` instance. + pub fn new(socket: PathBuf, network: CardanoNetwork, logger: Logger) -> Self { + Self { + socket, + network, + logger: logger.new_with_component_name::(), + phantom: PhantomData, + } + } + + /// Creates and returns a new `DmqClient` connected to the specified socket. + async fn new_client(&self) -> StdResult { + let magic = self.network.code(); + DmqClient::connect(&self.socket, magic) + .await + .map_err(|err| anyhow!(err)) + .with_context(|| "PallasChainReader failed to create a new client") + } + + /// Consume messages from the DMQ node. + pub async fn consume_messages(&self) -> StdResult> { + debug!(self.logger, "Waiting for messages from DMQ..."); + let mut client = self.new_client().await?; // TODO: add client cache + client + .msg_notification() + .send_request_messages_blocking() + .await + .map_err(|err| anyhow!("Failed to request notifications from DMQ server: {}", err))?; + + let reply = client.msg_notification().recv_next_reply().await.unwrap(); + debug!(self.logger, "Received single signatures from DMQ"; "messages" => ?reply); + if let Err(e) = client.msg_notification().send_done().await { + error!(self.logger, "Failed to send Done"; "error" => ?e); + } + + reply + .0 + .into_iter() + .map(|dmq_message| { + let opcert = OpCert::try_from_bytes(&dmq_message.operational_certificate) + .map_err(|e| anyhow!("Failed to parse operational certificate: {}", e))?; + let party_id = opcert.compute_protocol_party_id()?; + let payload = M::try_from_bytes(&dmq_message.msg_body) + .map_err(|e| anyhow!("Failed to parse DMQ message body: {}", e))?; + + Ok((payload, party_id)) + }) + .collect::>>() + .with_context(|| "Failed to parse DMQ messages") + } +} + +#[cfg(test)] +mod tests { + + use std::{fs, future, time::Duration, vec}; + + use mithril_common::{crypto_helper::TryToBytes, current_function, test_utils::TempDir}; + use pallas_network::miniprotocols::{localmsgnotification, localmsgsubmission::DmqMsg}; + use tokio::{net::UnixListener, task::JoinHandle, time::sleep}; + + use crate::{test::payload::DmqMessageTestPayload, test_tools::TestLogger}; + + use super::*; + + fn create_temp_dir(folder_name: &str) -> PathBuf { + TempDir::create_with_short_path("dmq_consumer", folder_name) + } + + fn fake_msgs() -> Vec { + vec![ + DmqMsg { + msg_id: vec![0, 1], + msg_body: DmqMessageTestPayload::new(b"msg_1").to_bytes_vec().unwrap(), + block_number: 10, + ttl: 100, + kes_signature: vec![0, 1, 2, 3], + operational_certificate: vec![ + 130, 132, 88, 32, 230, 80, 215, 83, 21, 9, 187, 108, 255, 215, 153, 140, 40, + 198, 142, 78, 200, 250, 98, 26, 9, 82, 32, 110, 161, 30, 176, 63, 205, 125, + 203, 41, 0, 0, 88, 64, 212, 171, 206, 39, 218, 5, 255, 3, 193, 52, 44, 198, + 171, 83, 19, 80, 114, 225, 186, 191, 156, 192, 84, 146, 245, 159, 31, 240, 9, + 247, 4, 87, 170, 168, 98, 199, 21, 139, 19, 190, 12, 251, 65, 215, 169, 26, 86, + 37, 137, 188, 17, 14, 178, 205, 175, 93, 39, 86, 4, 138, 187, 234, 95, 5, 88, + 32, 32, 253, 186, 201, 177, 11, 117, 135, 187, 167, 181, 188, 22, 59, 206, 105, + 231, 150, 215, 30, 78, 212, 76, 16, 252, 180, 72, 134, 137, 247, 161, 68, + ], + }, + DmqMsg { + msg_id: vec![1, 2], + msg_body: DmqMessageTestPayload::new(b"msg_2").to_bytes_vec().unwrap(), + block_number: 11, + ttl: 100, + kes_signature: vec![1, 2, 3, 4], + operational_certificate: vec![ + 130, 132, 88, 32, 230, 80, 215, 83, 21, 9, 187, 108, 255, 215, 153, 140, 40, + 198, 142, 78, 200, 250, 98, 26, 9, 82, 32, 110, 161, 30, 176, 63, 205, 125, + 203, 41, 0, 0, 88, 64, 132, 4, 199, 39, 190, 173, 88, 102, 121, 117, 55, 62, + 39, 189, 113, 96, 175, 24, 171, 240, 74, 42, 139, 202, 128, 185, 44, 130, 209, + 77, 191, 122, 196, 224, 33, 158, 187, 156, 203, 190, 173, 150, 247, 87, 172, + 58, 153, 185, 157, 87, 128, 14, 187, 107, 187, 215, 105, 195, 107, 135, 172, + 43, 173, 9, 88, 32, 77, 75, 24, 6, 47, 133, 2, 89, 141, 224, 69, 202, 123, 105, + 240, 103, 245, 159, 147, 177, 110, 58, 248, 115, 58, 152, 138, 220, 35, 65, + 245, 200, + ], + }, + ] + } + + fn setup_dmq_server(socket_path: PathBuf, reply_messages: Vec) -> JoinHandle<()> { + tokio::spawn({ + async move { + // server setup + if socket_path.exists() { + fs::remove_file(socket_path.clone()).unwrap(); + } + let listener = UnixListener::bind(socket_path).unwrap(); + let mut server = pallas_network::facades::DmqServer::accept(&listener, 0) + .await + .unwrap(); + + // init local msg notification server + let server_msg = server.msg_notification(); + + // server waits for blocking request from client + let request = server_msg.recv_next_request().await.unwrap(); + assert_eq!(request, localmsgnotification::Request::Blocking); + + if !reply_messages.is_empty() { + // server replies with messages if any + server_msg + .send_reply_messages_blocking(reply_messages) + .await + .unwrap(); + assert_eq!(*server_msg.state(), localmsgnotification::State::Idle); + + // server receives done from client + server_msg.recv_done().await.unwrap(); + assert_eq!(*server_msg.state(), localmsgnotification::State::Done); + } else { + // server waits if no message available + future::pending().await + } + } + }) + } + + #[tokio::test] + async fn pallas_dmq_consumer_publisher_succeeds_when_messages_are_available() { + let socket_path = create_temp_dir(current_function!()).join("node.socket"); + let reply_messages = fake_msgs(); + let server = setup_dmq_server(socket_path.clone(), reply_messages); + let client = tokio::spawn(async move { + let consumer = DmqConsumerPallas::new( + socket_path, + CardanoNetwork::TestNet(0), + TestLogger::stdout(), + ); + + consumer.consume_messages().await.unwrap() + }); + + let (_, messages) = tokio::join!(server, client); + + assert_eq!( + vec![ + ( + DmqMessageTestPayload::new(b"msg_1"), + "pool1mxyec46067n3querj9cxkk0g0zlag93pf3ya9vuyr3wgkq2e6t7".to_string() + ), + ( + DmqMessageTestPayload::new(b"msg_2"), + "pool17sln0evyk5tfj6zh2qrlk9vttgy6264sfe2fkec5mheasnlx3yd".to_string() + ), + ], + messages.unwrap() + ); + } + + #[tokio::test] + async fn pallas_dmq_consumer_publisher_blocks_when_no_message_available() { + let socket_path = create_temp_dir(current_function!()).join("node.socket"); + let reply_messages = vec![]; + let server = setup_dmq_server(socket_path.clone(), reply_messages); + let client = tokio::spawn(async move { + let consumer = DmqConsumerPallas::::new( + socket_path, + CardanoNetwork::TestNet(0), + TestLogger::stdout(), + ); + + consumer.consume_messages().await.unwrap(); + }); + + let result = tokio::select!( + _res = sleep(Duration::from_millis(100)) => {Err(anyhow!("Timeout"))}, + _res = client => {Ok(())}, + _res = server => {Ok(())}, + ); + + result.expect_err("Should have timed out"); + } +} diff --git a/internal/mithril-dmq-node/src/lib.rs b/internal/mithril-dmq-node/src/lib.rs index 6e34b58d5ba..5c473ef9ea7 100644 --- a/internal/mithril-dmq-node/src/lib.rs +++ b/internal/mithril-dmq-node/src/lib.rs @@ -1,10 +1,12 @@ #![warn(missing_docs)] //! This crate provides mechanisms to publish and consume messages of a Decentralized Message Queue network through a DMQ node. +mod consumer; mod message; mod publisher; pub mod test; +pub use consumer::DmqConsumerPallas; pub use message::DmqMessageBuilder; pub use publisher::DmqPublisherPallas; From e0d6f3efad0cac6436a5f2a49a0a043b65a5544c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jun 2025 18:20:52 +0200 Subject: [PATCH 06/38] feat(common): add 'RegisterSignatureMessageDmq' for DMQ single signatures --- .../src/entities/signed_entity_type.rs | 34 +++- mithril-common/src/messages/mod.rs | 2 +- .../src/messages/register_signature.rs | 163 +++++++++++++++++- 3 files changed, 194 insertions(+), 5 deletions(-) diff --git a/mithril-common/src/entities/signed_entity_type.rs b/mithril-common/src/entities/signed_entity_type.rs index a733061c5ea..68c56ec3f90 100644 --- a/mithril-common/src/entities/signed_entity_type.rs +++ b/mithril-common/src/entities/signed_entity_type.rs @@ -8,7 +8,10 @@ use serde::{Deserialize, Serialize}; use sha2::Sha256; use strum::{AsRefStr, Display, EnumDiscriminants, EnumIter, EnumString, IntoEnumIterator}; -use crate::StdResult; +use crate::{ + crypto_helper::{TryFromBytes, TryToBytes}, + StdResult, +}; use super::{BlockNumber, CardanoDbBeacon, Epoch}; @@ -154,6 +157,22 @@ impl SignedEntityType { } } +impl TryFromBytes for SignedEntityType { + fn try_from_bytes(bytes: &[u8]) -> StdResult { + let (res, _) = + bincode::serde::decode_from_slice::(bytes, bincode::config::standard()) + .map_err(|e| anyhow!(e))?; + + Ok(res) + } +} + +impl TryToBytes for SignedEntityType { + fn to_bytes_vec(&self) -> StdResult> { + bincode::serde::encode_to_vec(self, bincode::config::standard()).map_err(|e| e.into()) + } +} + impl SignedEntityTypeDiscriminants { /// Get all the discriminants pub fn all() -> BTreeSet { @@ -406,6 +425,19 @@ mod tests { ); } + #[test] + fn bytes_encoding() { + let cardano_stake_distribution = SignedEntityType::CardanoStakeDistribution(Epoch(25)); + let cardano_stake_distribution_bytes = cardano_stake_distribution.to_bytes_vec().unwrap(); + let cardano_stake_distribution_from_bytes = + SignedEntityType::try_from_bytes(&cardano_stake_distribution_bytes).unwrap(); + + assert_eq!( + cardano_stake_distribution, + cardano_stake_distribution_from_bytes + ); + } + // Expected ord: // MithrilStakeDistribution < CardanoStakeDistribution < CardanoImmutableFilesFull < CardanoDatabase < CardanoTransactions #[test] diff --git a/mithril-common/src/messages/mod.rs b/mithril-common/src/messages/mod.rs index 1de4ccee9ce..d25b57103ab 100644 --- a/mithril-common/src/messages/mod.rs +++ b/mithril-common/src/messages/mod.rs @@ -61,7 +61,7 @@ pub use mithril_stake_distribution::MithrilStakeDistributionMessage; pub use mithril_stake_distribution_list::{ MithrilStakeDistributionListItemMessage, MithrilStakeDistributionListMessage, }; -pub use register_signature::RegisterSignatureMessageHttp; +pub use register_signature::{RegisterSignatureMessageDmq, RegisterSignatureMessageHttp}; pub use register_signer::RegisterSignerMessage; pub use snapshot::SnapshotMessage; pub use snapshot_download::SnapshotDownloadMessage; diff --git a/mithril-common/src/messages/register_signature.rs b/mithril-common/src/messages/register_signature.rs index 3a595cbfddb..724233bfce3 100644 --- a/mithril-common/src/messages/register_signature.rs +++ b/mithril-common/src/messages/register_signature.rs @@ -1,7 +1,13 @@ -use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Formatter}; -use crate::entities::{HexEncodedSingleSignature, LotteryIndex, PartyId, SignedEntityType}; +use anyhow::anyhow; +use serde::{Deserialize, Serialize}; + +use crate::{ + crypto_helper::{ProtocolSingleSignature, TryFromBytes, TryToBytes}, + entities::{HexEncodedSingleSignature, LotteryIndex, PartyId, SignedEntityType}, + StdResult, +}; #[cfg(any(test, feature = "test_tools"))] use crate::test_utils::fake_keys; @@ -65,11 +71,122 @@ impl Debug for RegisterSignatureMessageHttp { } } +/// Message structure to register single signature through the DMQ network. +#[derive(Clone, PartialEq, Eq)] +pub struct RegisterSignatureMessageDmq { + /// Signed entity type + pub signed_entity_type: SignedEntityType, + + /// The single signature. + pub signature: ProtocolSingleSignature, +} + +impl RegisterSignatureMessageDmq { + cfg_test_tools! { + /// Return a dummy test entity (test-only). + pub fn dummy() -> Self { + use crate::entities::Epoch; + Self { + signed_entity_type: SignedEntityType::MithrilStakeDistribution(Epoch(5)), + signature: fake_keys::single_signature()[0].try_into().unwrap(), + } + } + } + + /// Convert an `RegisterSignatureMessageDmq` into bytes + /// + /// # Layout + /// * Signed entity type length (u16) + /// * Signed entity type + /// * Protocol signature length (u32) + /// * Protocol signature + pub fn try_to_bytes_vec(&self) -> StdResult> { + let mut bytes = Vec::new(); + + let signed_entity_bytes = self.signed_entity_type.to_bytes_vec()?; + bytes.extend_from_slice(&(signed_entity_bytes.len() as u16).to_be_bytes()); + bytes.extend_from_slice(&signed_entity_bytes); + + let signature_bytes = self.signature.to_bytes_vec()?; + bytes.extend_from_slice(&(signature_bytes.len() as u32).to_be_bytes()); + bytes.extend_from_slice(&signature_bytes); + + Ok(bytes) + } + + /// Extract a `RegisterSignatureMessageDmq` from bytes. + pub fn try_from_bytes_vec(bytes: &[u8]) -> StdResult { + let mut bytes_index = 0; + + let mut u16_bytes = [0u8; 2]; + u16_bytes.copy_from_slice( + bytes + .get(bytes_index..bytes_index + 2) + .ok_or(anyhow!("Failed to read `Signed entity type length` bytes"))?, + ); + let signed_entity_bytes_length = u16::from_be_bytes(u16_bytes) as usize; + bytes_index += 2; + + let signed_entity_bytes = bytes + .get(bytes_index..bytes_index + signed_entity_bytes_length) + .ok_or(anyhow!("Failed to read `Signed entity type` bytes"))?; + let signed_entity_type = SignedEntityType::try_from_bytes(signed_entity_bytes)?; + bytes_index += signed_entity_bytes_length; + + let mut u32_bytes = [0u8; 4]; + u32_bytes.copy_from_slice( + bytes + .get(bytes_index..bytes_index + 4) + .ok_or(anyhow!("Failed to read `Signature length` bytes"))?, + ); + let signature_bytes_length = u32::from_be_bytes(u32_bytes) as usize; + bytes_index += 4; + + let signature_bytes = bytes + .get(bytes_index..bytes_index + signature_bytes_length) + .ok_or(anyhow!("Failed to read `Signature` bytes"))?; + let signature = ProtocolSingleSignature::from_bytes(signature_bytes)?; + + Ok(Self { + signed_entity_type, + signature, + }) + } +} + +impl Debug for RegisterSignatureMessageDmq { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let is_pretty_printing = f.alternate(); + let mut debug = f.debug_struct("RegisterSignatureMessageDmq"); + debug.field( + "signed_entity_type", + &format_args!("{:?}", self.signed_entity_type), + ); + + match is_pretty_printing { + true => debug.field("signature", &self.signature).finish(), + false => debug.finish_non_exhaustive(), + } + } +} + +impl TryFromBytes for RegisterSignatureMessageDmq { + fn try_from_bytes(bytes: &[u8]) -> StdResult { + Self::try_from_bytes_vec(bytes) + } +} + +impl TryToBytes for RegisterSignatureMessageDmq { + fn to_bytes_vec(&self) -> StdResult> { + self.try_to_bytes_vec() + } +} + #[cfg(test)] mod tests { use super::*; - mod http_message { + mod golden_http_message { use crate::entities::{CardanoDbBeacon, Epoch}; use super::*; @@ -127,4 +244,44 @@ mod tests { assert_eq!(golden_message_current(), message); } } + + mod golden_dmq_message { + use hex::FromHex; + + use crate::entities::{CardanoDbBeacon, Epoch}; + + use super::*; + + const CURRENT_BYTES_HEX: &str = r#"0005020afbc006000001b8000000000000002f0000000000000000000000000000000100000000000000030000000000000004000000000000000600000000000000080000000000000009000000000000000a000000000000000b000000000000000c000000000000000e00000000000000120000000000000015000000000000001600000000000000170000000000000019000000000000001a000000000000001b000000000000001e0000000000000021000000000000002200000000000000260000000000000029000000000000002b0000000000000032000000000000003a000000000000003b000000000000003c000000000000003d000000000000003e0000000000000043000000000000004500000000000000470000000000000049000000000000004b000000000000004c000000000000004d0000000000000051000000000000005200000000000000530000000000000054000000000000005a000000000000005b000000000000005c000000000000005d0000000000000061000000000000006282b10fe518fbf7abc4d28f7156bd5c387021c1d436d61cd8e3ad647fb22862571db5ff6f9de0eb2e64a9e8bdfc528b240000000000000002"#; + + fn golden_message_current() -> RegisterSignatureMessageDmq { + RegisterSignatureMessageDmq { + signed_entity_type: SignedEntityType::CardanoImmutableFilesFull( + CardanoDbBeacon::new(*Epoch(10), 1728), + ), + signature: "7b227369676d61223a5b3133302c3137372c31352c3232392c32342c3235312c3234372c3137312c3139362c3231302c3134332c3131332c38362c3138392c39322c35362c3131322c33332c3139332c3231322c35342c3231342c32382c3231362c3232372c3137332c3130302c3132372c3137382c34302c39382c38372c32392c3138312c3235352c3131312c3135372c3232342c3233352c34362c3130302c3136392c3233322c3138392c3235322c38322c3133392c33365d2c22696e6465786573223a5b302c312c332c342c362c382c392c31302c31312c31322c31342c31382c32312c32322c32332c32352c32362c32372c33302c33332c33342c33382c34312c34332c35302c35382c35392c36302c36312c36322c36372c36392c37312c37332c37352c37362c37372c38312c38322c38332c38342c39302c39312c39322c39332c39372c39385d2c227369676e65725f696e646578223a327d".to_string().try_into().unwrap(), + } + } + + #[test] + fn test_current_bytes_decoded_into_current_message() { + let message_from_bytes_hex = RegisterSignatureMessageDmq::try_from_bytes_vec( + &Vec::from_hex(CURRENT_BYTES_HEX).unwrap(), + ) + .unwrap(); + + assert_eq!(golden_message_current(), message_from_bytes_hex); + } + + #[test] + fn test_current_bijective_bytes_codec() { + let message_to_bytes = golden_message_current().try_to_bytes_vec().unwrap(); + let message_from_bytes = + RegisterSignatureMessageDmq::try_from_bytes_vec(&message_to_bytes).unwrap(); + let message_from_bytes_to_bytes = message_from_bytes.try_to_bytes_vec().unwrap(); + + assert_eq!(golden_message_current(), message_from_bytes); + assert_eq!(message_to_bytes, message_from_bytes_to_bytes); + } + } } From 108134eea4c051acd615963ffadcdc9f4039d75e Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jun 2025 18:22:14 +0200 Subject: [PATCH 07/38] feat(signer): add Pallas based 'SignaturePublisher' implementation --- mithril-signer/Cargo.toml | 1 + .../src/services/signature_publisher/mod.rs | 2 ++ .../services/signature_publisher/pallas.rs | 29 +++++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 mithril-signer/src/services/signature_publisher/pallas.rs diff --git a/mithril-signer/Cargo.toml b/mithril-signer/Cargo.toml index 8c2e6247bb8..5aad717e079 100644 --- a/mithril-signer/Cargo.toml +++ b/mithril-signer/Cargo.toml @@ -26,6 +26,7 @@ mithril-cardano-node-chain = { path = "../internal/cardano-node/mithril-cardano- mithril-cardano-node-internal-database = { path = "../internal/cardano-node/mithril-cardano-node-internal-database" } mithril-cli-helper = { path = "../internal/mithril-cli-helper" } mithril-common = { path = "../mithril-common", features = ["full"] } +mithril-dmq-node = { path = "../internal/mithril-dmq-node" } mithril-doc = { path = "../internal/mithril-doc" } mithril-era = { path = "../internal/mithril-era" } mithril-metric = { path = "../internal/mithril-metric" } diff --git a/mithril-signer/src/services/signature_publisher/mod.rs b/mithril-signer/src/services/signature_publisher/mod.rs index ad86537109a..09a37635148 100644 --- a/mithril-signer/src/services/signature_publisher/mod.rs +++ b/mithril-signer/src/services/signature_publisher/mod.rs @@ -2,9 +2,11 @@ mod delayer; mod http; mod interface; mod noop; +mod pallas; mod retrier; pub use delayer::*; pub use interface::*; pub use noop::*; +pub use pallas::*; pub use retrier::*; diff --git a/mithril-signer/src/services/signature_publisher/pallas.rs b/mithril-signer/src/services/signature_publisher/pallas.rs new file mode 100644 index 00000000000..f3ffd189c63 --- /dev/null +++ b/mithril-signer/src/services/signature_publisher/pallas.rs @@ -0,0 +1,29 @@ +use anyhow::Context; +use async_trait::async_trait; + +use mithril_common::{ + entities::{ProtocolMessage, SignedEntityType, SingleSignature}, + messages::RegisterSignatureMessageDmq, + StdResult, +}; +use mithril_dmq_node::DmqPublisherPallas; + +use super::SignaturePublisher; + +#[async_trait] +impl SignaturePublisher for DmqPublisherPallas { + async fn publish( + &self, + signed_entity_type: &SignedEntityType, + signature: &SingleSignature, + _protocol_message: &ProtocolMessage, + ) -> StdResult<()> { + let message = RegisterSignatureMessageDmq { + signature: signature.signature.to_owned(), + signed_entity_type: signed_entity_type.to_owned(), + }; + self.publish_message(message) + .await + .with_context(|| "Failed to publish DMQ message") + } +} From 64f1ad28431bc630720ba2e4f75ed47f11e53047 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jun 2025 18:22:37 +0200 Subject: [PATCH 08/38] feat(aggregator): add Pallas based 'SignatureConsumer' implementation --- Cargo.lock | 2 + mithril-aggregator/Cargo.toml | 1 + .../src/services/signature_consumer/mod.rs | 2 + .../src/services/signature_consumer/pallas.rs | 39 +++++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 mithril-aggregator/src/services/signature_consumer/pallas.rs diff --git a/Cargo.lock b/Cargo.lock index 96967db0ea9..8f597b9ccd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3929,6 +3929,7 @@ dependencies = [ "mithril-cardano-node-internal-database", "mithril-cli-helper", "mithril-common", + "mithril-dmq-node", "mithril-doc", "mithril-era", "mithril-metric", @@ -4382,6 +4383,7 @@ dependencies = [ "mithril-cardano-node-internal-database", "mithril-cli-helper", "mithril-common", + "mithril-dmq-node", "mithril-doc", "mithril-era", "mithril-metric", diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index ddf7d805559..d0b2f87f550 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -29,6 +29,7 @@ mithril-cardano-node-chain = { path = "../internal/cardano-node/mithril-cardano- mithril-cardano-node-internal-database = { path = "../internal/cardano-node/mithril-cardano-node-internal-database" } mithril-cli-helper = { path = "../internal/mithril-cli-helper" } mithril-common = { path = "../mithril-common", features = ["full"] } +mithril-dmq-node = { path = "../internal/mithril-dmq-node" } mithril-doc = { path = "../internal/mithril-doc" } mithril-era = { path = "../internal/mithril-era" } mithril-metric = { path = "../internal/mithril-metric" } diff --git a/mithril-aggregator/src/services/signature_consumer/mod.rs b/mithril-aggregator/src/services/signature_consumer/mod.rs index 96610b2b029..7ed25404854 100644 --- a/mithril-aggregator/src/services/signature_consumer/mod.rs +++ b/mithril-aggregator/src/services/signature_consumer/mod.rs @@ -1,7 +1,9 @@ mod fake; mod interface; mod noop; +mod pallas; pub use fake::*; pub use interface::*; pub use noop::*; +pub use pallas::*; diff --git a/mithril-aggregator/src/services/signature_consumer/pallas.rs b/mithril-aggregator/src/services/signature_consumer/pallas.rs new file mode 100644 index 00000000000..a1ca9cbc15b --- /dev/null +++ b/mithril-aggregator/src/services/signature_consumer/pallas.rs @@ -0,0 +1,39 @@ +use anyhow::Context; +use async_trait::async_trait; + +use mithril_common::{ + entities::{SignedEntityType, SingleSignature}, + messages::RegisterSignatureMessageDmq, + StdResult, +}; + +use mithril_dmq_node::DmqConsumerPallas; + +use super::SignatureConsumer; + +#[async_trait] +impl SignatureConsumer for DmqConsumerPallas { + async fn get_signatures(&self) -> StdResult> { + self.consume_messages() + .await + .map(|messages| { + messages + .into_iter() + .map(|(message, party_id)| { + let signature = message.signature; + let won_indexes = signature.indexes.clone(); + let single_signature = + SingleSignature::new(party_id, signature, won_indexes); + let signed_entity_type = message.signed_entity_type; + + (single_signature, signed_entity_type) + }) + .collect() + }) + .with_context(|| "Failed to get signatures from DMQ") + } + + fn get_origin_tag(&self) -> String { + "DMQ".to_string() + } +} From e428259305570f4fd635a2602c238607d0d5fc81 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jun 2025 18:52:27 +0200 Subject: [PATCH 09/38] feat(dmq): add a 'DmqConsumer' trait Which is implemented with Pallas in 'DmqConsumerPallas'. --- Cargo.lock | 28 ++++++++++--------- internal/mithril-dmq-node/Cargo.toml | 2 ++ .../src/consumer/interface.rs | 11 ++++++++ internal/mithril-dmq-node/src/consumer/mod.rs | 5 ++++ .../src/{consumer.rs => consumer/pallas.rs} | 9 ++++-- internal/mithril-dmq-node/src/lib.rs | 2 +- 6 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 internal/mithril-dmq-node/src/consumer/interface.rs create mode 100644 internal/mithril-dmq-node/src/consumer/mod.rs rename internal/mithril-dmq-node/src/{consumer.rs => consumer/pallas.rs} (97%) diff --git a/Cargo.lock b/Cargo.lock index 8f597b9ccd1..63a2d2b5b16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3869,7 +3869,7 @@ checksum = "a9882ef5c56df184b8ffc107fc6c61e33ee3a654b021961d790a78571bb9d67a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -4021,9 +4021,9 @@ dependencies = [ "mockall", "nom 8.0.0", "pallas-addresses", - "pallas-codec 0.32.0", - "pallas-crypto 0.32.0", - "pallas-network 0.32.0", + "pallas-codec 0.32.1", + "pallas-crypto 0.32.1", + "pallas-network 0.32.1", "pallas-primitives", "pallas-traverse", "rand_core 0.6.4", @@ -4205,9 +4205,11 @@ name = "mithril-dmq-node" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "blake2 0.10.6", "mithril-cardano-node-chain", "mithril-common", + "mockall", "pallas-network 1.0.0-alpha.2", "slog", "slog-async", @@ -4968,8 +4970,8 @@ dependencies = [ "crc", "cryptoxide", "hex", - "pallas-codec 0.32.0", - "pallas-crypto 0.32.0", + "pallas-codec 0.32.1", + "pallas-crypto 0.32.1", "thiserror 1.0.69", ] @@ -5004,7 +5006,7 @@ checksum = "59c89ea16190a87a1d8bd36923093740a2b659ed6129f4636329319a70cc4db3" dependencies = [ "cryptoxide", "hex", - "pallas-codec 0.32.0", + "pallas-codec 0.32.1", "rand_core 0.6.4", "serde", "thiserror 1.0.69", @@ -5033,8 +5035,8 @@ dependencies = [ "byteorder", "hex", "itertools 0.13.0", - "pallas-codec 0.32.0", - "pallas-crypto 0.32.0", + "pallas-codec 0.32.1", + "pallas-crypto 0.32.1", "rand 0.8.5", "socket2", "thiserror 1.0.69", @@ -5069,8 +5071,8 @@ dependencies = [ "bech32 0.9.1", "hex", "log", - "pallas-codec 0.32.0", - "pallas-crypto 0.32.0", + "pallas-codec 0.32.1", + "pallas-crypto 0.32.1", "serde", "serde_json", ] @@ -5084,8 +5086,8 @@ dependencies = [ "hex", "itertools 0.13.0", "pallas-addresses", - "pallas-codec 0.32.0", - "pallas-crypto 0.32.0", + "pallas-codec 0.32.1", + "pallas-crypto 0.32.1", "pallas-primitives", "paste", "serde", diff --git a/internal/mithril-dmq-node/Cargo.toml b/internal/mithril-dmq-node/Cargo.toml index af6e6c9726c..7f02687e442 100644 --- a/internal/mithril-dmq-node/Cargo.toml +++ b/internal/mithril-dmq-node/Cargo.toml @@ -15,6 +15,7 @@ crate-type = ["lib", "cdylib", "staticlib"] [dependencies] anyhow = { workspace = true } +async-trait = { workspace = true } blake2 = "0.10.6" mithril-cardano-node-chain = { path = "../cardano-node/mithril-cardano-node-chain" } mithril-common = { path = "../../mithril-common" } @@ -24,5 +25,6 @@ tokio = { workspace = true } [dev-dependencies] mithril-common = { path = "../../mithril-common", features = ["test_tools"] } +mockall = { workspace = true } slog-async = { workspace = true } slog-term = { workspace = true } \ No newline at end of file diff --git a/internal/mithril-dmq-node/src/consumer/interface.rs b/internal/mithril-dmq-node/src/consumer/interface.rs new file mode 100644 index 00000000000..afad017a47b --- /dev/null +++ b/internal/mithril-dmq-node/src/consumer/interface.rs @@ -0,0 +1,11 @@ +use std::fmt::Debug; + +use mithril_common::{crypto_helper::TryFromBytes, entities::PartyId, StdResult}; + +/// Trait for consuming messages from a DMQ node. +#[cfg_attr(test, mockall::automock)] +#[async_trait::async_trait] +pub trait DmqConsumer: Send + Sync { + /// Consume messages from the DMQ node. + async fn consume_messages(&self) -> StdResult>; +} diff --git a/internal/mithril-dmq-node/src/consumer/mod.rs b/internal/mithril-dmq-node/src/consumer/mod.rs new file mode 100644 index 00000000000..4035f6c0659 --- /dev/null +++ b/internal/mithril-dmq-node/src/consumer/mod.rs @@ -0,0 +1,5 @@ +mod interface; +mod pallas; + +pub use interface::*; +pub use pallas::*; diff --git a/internal/mithril-dmq-node/src/consumer.rs b/internal/mithril-dmq-node/src/consumer/pallas.rs similarity index 97% rename from internal/mithril-dmq-node/src/consumer.rs rename to internal/mithril-dmq-node/src/consumer/pallas.rs index e1d3ddc48bd..005aae69a48 100644 --- a/internal/mithril-dmq-node/src/consumer.rs +++ b/internal/mithril-dmq-node/src/consumer/pallas.rs @@ -11,7 +11,10 @@ use mithril_common::{ CardanoNetwork, StdResult, }; +use crate::DmqConsumer; + /// A DMQ consumer implementation. +/// /// This implementation is built upon the n2c mini-protocols DMQ implementation in Pallas. pub struct DmqConsumerPallas { socket: PathBuf, @@ -39,9 +42,11 @@ impl DmqConsumerPallas { .map_err(|err| anyhow!(err)) .with_context(|| "PallasChainReader failed to create a new client") } +} - /// Consume messages from the DMQ node. - pub async fn consume_messages(&self) -> StdResult> { +#[async_trait::async_trait] +impl DmqConsumer for DmqConsumerPallas { + async fn consume_messages(&self) -> StdResult> { debug!(self.logger, "Waiting for messages from DMQ..."); let mut client = self.new_client().await?; // TODO: add client cache client diff --git a/internal/mithril-dmq-node/src/lib.rs b/internal/mithril-dmq-node/src/lib.rs index 5c473ef9ea7..8b6d1a71ae6 100644 --- a/internal/mithril-dmq-node/src/lib.rs +++ b/internal/mithril-dmq-node/src/lib.rs @@ -6,7 +6,7 @@ mod message; mod publisher; pub mod test; -pub use consumer::DmqConsumerPallas; +pub use consumer::{DmqConsumer, DmqConsumerPallas}; pub use message::DmqMessageBuilder; pub use publisher::DmqPublisherPallas; From bb2111809a1186477c29fb343d27977fa9681488 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jun 2025 18:53:01 +0200 Subject: [PATCH 10/38] feat(aggregator): update 'SignatureConsumer' implementation --- mithril-aggregator/src/services/signature_consumer/pallas.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mithril-aggregator/src/services/signature_consumer/pallas.rs b/mithril-aggregator/src/services/signature_consumer/pallas.rs index a1ca9cbc15b..ceb2d2f141e 100644 --- a/mithril-aggregator/src/services/signature_consumer/pallas.rs +++ b/mithril-aggregator/src/services/signature_consumer/pallas.rs @@ -7,7 +7,7 @@ use mithril_common::{ StdResult, }; -use mithril_dmq_node::DmqConsumerPallas; +use mithril_dmq_node::{DmqConsumer, DmqConsumerPallas}; use super::SignatureConsumer; From e88d9dc6048bcbcbf419870a42989a4d9e518015 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jun 2025 19:11:48 +0200 Subject: [PATCH 11/38] feat(dmq): add a 'DmqPublisher' trait Which is implemented with Pallas in 'DmqPublisherPallas'. --- internal/mithril-dmq-node/src/lib.rs | 2 +- internal/mithril-dmq-node/src/publisher/interface.rs | 11 +++++++++++ internal/mithril-dmq-node/src/publisher/mod.rs | 5 +++++ .../src/{publisher.rs => publisher/pallas.rs} | 11 +++++++---- 4 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 internal/mithril-dmq-node/src/publisher/interface.rs create mode 100644 internal/mithril-dmq-node/src/publisher/mod.rs rename internal/mithril-dmq-node/src/{publisher.rs => publisher/pallas.rs} (96%) diff --git a/internal/mithril-dmq-node/src/lib.rs b/internal/mithril-dmq-node/src/lib.rs index 8b6d1a71ae6..db7536fc64e 100644 --- a/internal/mithril-dmq-node/src/lib.rs +++ b/internal/mithril-dmq-node/src/lib.rs @@ -8,7 +8,7 @@ pub mod test; pub use consumer::{DmqConsumer, DmqConsumerPallas}; pub use message::DmqMessageBuilder; -pub use publisher::DmqPublisherPallas; +pub use publisher::{DmqPublisher, DmqPublisherPallas}; #[cfg(test)] pub(crate) mod test_tools { diff --git a/internal/mithril-dmq-node/src/publisher/interface.rs b/internal/mithril-dmq-node/src/publisher/interface.rs new file mode 100644 index 00000000000..4796cc0204a --- /dev/null +++ b/internal/mithril-dmq-node/src/publisher/interface.rs @@ -0,0 +1,11 @@ +use std::fmt::Debug; + +use mithril_common::{crypto_helper::TryToBytes, StdResult}; + +/// Trait for publishing messages from a DMQ node. +#[cfg_attr(test, mockall::automock)] +#[async_trait::async_trait] +pub trait DmqPublisher: Send + Sync { + /// Publishes a message to the DMQ node. + async fn publish_message(&self, message: M) -> StdResult<()>; +} diff --git a/internal/mithril-dmq-node/src/publisher/mod.rs b/internal/mithril-dmq-node/src/publisher/mod.rs new file mode 100644 index 00000000000..4035f6c0659 --- /dev/null +++ b/internal/mithril-dmq-node/src/publisher/mod.rs @@ -0,0 +1,5 @@ +mod interface; +mod pallas; + +pub use interface::*; +pub use pallas::*; diff --git a/internal/mithril-dmq-node/src/publisher.rs b/internal/mithril-dmq-node/src/publisher/pallas.rs similarity index 96% rename from internal/mithril-dmq-node/src/publisher.rs rename to internal/mithril-dmq-node/src/publisher/pallas.rs index f3466e00cf0..5eb16399f9c 100644 --- a/internal/mithril-dmq-node/src/publisher.rs +++ b/internal/mithril-dmq-node/src/publisher/pallas.rs @@ -8,9 +8,10 @@ use mithril_common::{ crypto_helper::TryToBytes, logging::LoggerExtensions, CardanoNetwork, StdResult, }; -use crate::DmqMessageBuilder; +use crate::{DmqMessageBuilder, DmqPublisher}; /// A DMQ publisher implementation. +/// /// This implementation is built upon the n2c mini-protocols DMQ implementation in Pallas. pub struct DmqPublisherPallas { socket: PathBuf, @@ -45,9 +46,11 @@ impl DmqPublisherPallas { .map_err(|err| anyhow!(err)) .with_context(|| "PallasChainReader failed to create a new client") } +} - /// Publishes a message to the DMQ node. - pub async fn publish_message(&self, message: M) -> StdResult<()> { +#[async_trait::async_trait] +impl DmqPublisher for DmqPublisherPallas { + async fn publish_message(&self, message: M) -> StdResult<()> { debug!( self.logger, "Publish message to DMQ"; @@ -78,7 +81,7 @@ impl DmqPublisherPallas { } } -#[cfg(test)] +#[cfg(all(test, unix))] mod tests { use std::{fs, sync::Arc}; From 2e33bbcff5d84676fd268464e8283bd88a9f353e Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jun 2025 19:12:22 +0200 Subject: [PATCH 12/38] feat(signer): update 'SignaturePublisher' implementation --- mithril-signer/src/services/signature_publisher/pallas.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mithril-signer/src/services/signature_publisher/pallas.rs b/mithril-signer/src/services/signature_publisher/pallas.rs index f3ffd189c63..820df837617 100644 --- a/mithril-signer/src/services/signature_publisher/pallas.rs +++ b/mithril-signer/src/services/signature_publisher/pallas.rs @@ -6,7 +6,7 @@ use mithril_common::{ messages::RegisterSignatureMessageDmq, StdResult, }; -use mithril_dmq_node::DmqPublisherPallas; +use mithril_dmq_node::{DmqPublisher, DmqPublisherPallas}; use super::SignaturePublisher; From ee72e2f99ace095cb9f24a197a70b3273ce5245a Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 24 Jun 2025 17:34:54 +0200 Subject: [PATCH 13/38] feat(common): add 'KesSignerFake' implementation in 'kes' module --- .../crypto_helper/cardano/kes/fake_signer.rs | 94 +++++++++++++++++++ .../src/crypto_helper/cardano/kes/mod.rs | 2 + 2 files changed, 96 insertions(+) create mode 100644 mithril-common/src/crypto_helper/cardano/kes/fake_signer.rs diff --git a/mithril-common/src/crypto_helper/cardano/kes/fake_signer.rs b/mithril-common/src/crypto_helper/cardano/kes/fake_signer.rs new file mode 100644 index 00000000000..2a930045806 --- /dev/null +++ b/mithril-common/src/crypto_helper/cardano/kes/fake_signer.rs @@ -0,0 +1,94 @@ +use std::collections::VecDeque; + +use kes_summed_ed25519::kes::Sum6KesSig; +use std::sync::Mutex; + +use crate::{ + crypto_helper::{ + cardano::{create_kes_cryptographic_material, KesSignerStandard}, + KESPeriod, KesSigner, OpCert, + }, + StdResult, +}; + +type KesSignatureResult = StdResult<(Sum6KesSig, OpCert)>; + +/// Fake KES Signer implementation. +pub struct FakeKesSigner { + results: Mutex>, +} + +impl FakeKesSigner { + /// Creates a new `FakeSignatureConsumer` instance. + pub fn new(results: Vec) -> Self { + Self { + results: Mutex::new(results.into()), + } + } + + /// Returns a dummy signature result that is always successful. + pub fn dummy_signature() -> (Sum6KesSig, OpCert) { + let (_party_id, operational_certificate_file, kes_secret_key_file) = + create_kes_cryptographic_material( + 1, + 0 as KESPeriod, + "fake_kes_signer_returns_signature_batches_in_expected_order", + ); + let message = b"Test message for KES signing"; + let kes_signer = KesSignerStandard::new(kes_secret_key_file, operational_certificate_file); + let kes_signing_period = 1; + let (kes_signature, op_cert) = kes_signer + .sign(message, kes_signing_period) + .expect("Signing should not fail"); + + (kes_signature, op_cert) + } + + /// Returns a dummy signature result that always fails. + pub fn dummy_signature_result_err() -> KesSignatureResult { + Err(anyhow::anyhow!("Dummy error")) + } +} + +impl KesSigner for FakeKesSigner { + fn sign(&self, _message: &[u8], _kes_period: KESPeriod) -> KesSignatureResult { + let mut results = self.results.lock().unwrap(); + + results.pop_front().unwrap() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fake_kes_signer_returns_signature_batches_in_expected_order() { + let (_party_id, operational_certificate_file, kes_secret_key_file) = + create_kes_cryptographic_material( + 1, + 0 as KESPeriod, + "fake_kes_signer_returns_signature_batches_in_expected_order", + ); + let message = b"Test message for KES signing"; + let kes_signer = KesSignerStandard::new(kes_secret_key_file, operational_certificate_file); + let kes_signing_period = 1; + let (kes_signature, op_cert) = kes_signer + .sign(message, kes_signing_period) + .expect("Signing should not fail"); + let fake_kes_signer = FakeKesSigner::new(vec![ + Ok((kes_signature, op_cert.clone())), + Err(anyhow::anyhow!("Fake error")), + ]); + + let (kes_signature_1, op_cert_1) = fake_kes_signer + .sign(message, kes_signing_period) + .expect("Signing should not fail"); + assert_eq!(kes_signature, kes_signature_1); + assert_eq!(op_cert, op_cert_1); + + fake_kes_signer + .sign(message, kes_signing_period) + .expect_err("Signing should fail"); + } +} diff --git a/mithril-common/src/crypto_helper/cardano/kes/mod.rs b/mithril-common/src/crypto_helper/cardano/kes/mod.rs index 1dc2cc27ba3..b8f0bcc2314 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/mod.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/mod.rs @@ -1,4 +1,5 @@ mod error; +mod fake_signer; mod interface; mod signer_with_key; #[cfg(test)] @@ -7,6 +8,7 @@ pub(crate) mod tests_setup; mod verifier_standard; pub use error::*; +pub use fake_signer::*; pub use interface::*; pub use signer_with_key::*; pub use verifier_standard::*; From c729db023e38b8b4bc4171b31f02bdee7f28f1d6 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Jun 2025 11:54:09 +0200 Subject: [PATCH 14/38] test(dmq): add 'DmqConsumerFake' test double --- .../src/test/double/consumer.rs | 81 +++++++++++++++++++ .../mithril-dmq-node/src/test/double/mod.rs | 7 ++ internal/mithril-dmq-node/src/test/mod.rs | 3 + 3 files changed, 91 insertions(+) create mode 100644 internal/mithril-dmq-node/src/test/double/consumer.rs create mode 100644 internal/mithril-dmq-node/src/test/double/mod.rs diff --git a/internal/mithril-dmq-node/src/test/double/consumer.rs b/internal/mithril-dmq-node/src/test/double/consumer.rs new file mode 100644 index 00000000000..612b9e73141 --- /dev/null +++ b/internal/mithril-dmq-node/src/test/double/consumer.rs @@ -0,0 +1,81 @@ +use std::{collections::VecDeque, fmt::Debug}; + +use tokio::sync::Mutex; + +use mithril_common::{crypto_helper::TryFromBytes, entities::PartyId, StdResult}; + +use crate::DmqConsumer; + +type ConsumerReturn = StdResult>; + +/// A fake implementation of the [DmqConsumer] trait for testing purposes. +pub struct DmqConsumerFake { + results: Mutex>>, +} + +impl DmqConsumerFake { + /// Creates a new `DmqConsumerFake` instance with the provided results. + pub fn new(results: Vec>>) -> Self { + Self { + results: Mutex::new(VecDeque::from(results)), + } + } +} + +#[async_trait::async_trait] +impl DmqConsumer for DmqConsumerFake { + async fn consume_messages(&self) -> ConsumerReturn { + let mut results = self.results.lock().await; + + results + .pop_front() + .ok_or_else(|| anyhow::anyhow!("No more results available in DmqConsumerFake"))? + } +} + +#[cfg(test)] +mod tests { + use crate::test::payload::DmqMessageTestPayload; + + use super::*; + + #[tokio::test] + async fn consume_messages_success() { + let consumer = DmqConsumerFake::new(vec![ + Ok(vec![( + DmqMessageTestPayload::new(b"test-1"), + "pool-id-1".to_string(), + )]), + Ok(vec![( + DmqMessageTestPayload::new(b"test-2"), + "pool-id-2".to_string(), + )]), + ]); + + let messages = consumer.consume_messages().await.unwrap(); + + assert_eq!( + vec![( + DmqMessageTestPayload::new(b"test-1"), + "pool-id-1".to_string(), + )], + messages + ); + } + + #[tokio::test] + async fn consume_messages_failure() { + let consumer = DmqConsumerFake::new(vec![ + Err(anyhow::anyhow!("Test error")), + Ok(vec![( + DmqMessageTestPayload::new(b"test-2"), + "pool-id-2".to_string(), + )]), + ]); + + consumer + .consume_messages() + .await + .expect_err("DmqConsumerFake should return an error"); + } +} diff --git a/internal/mithril-dmq-node/src/test/double/mod.rs b/internal/mithril-dmq-node/src/test/double/mod.rs new file mode 100644 index 00000000000..5d3d2b87672 --- /dev/null +++ b/internal/mithril-dmq-node/src/test/double/mod.rs @@ -0,0 +1,7 @@ +//! Test doubles +//! +//! Enable unit testing with controlled inputs and predictable behavior. + +mod consumer; + +pub use consumer::*; diff --git a/internal/mithril-dmq-node/src/test/mod.rs b/internal/mithril-dmq-node/src/test/mod.rs index 0363df24048..be8d671b1aa 100644 --- a/internal/mithril-dmq-node/src/test/mod.rs +++ b/internal/mithril-dmq-node/src/test/mod.rs @@ -1,5 +1,8 @@ //! Test utilities. //! //! ⚠ Do not use in production code ⚠ +//! +//! This module provides in particular test doubles for the traits defined in this crate. +pub mod double; pub(crate) mod payload; From e50c77a2e753725090f7846e17bc03bf48f863bc Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Jun 2025 14:29:57 +0200 Subject: [PATCH 15/38] test(dmq): add 'DmqPublisherFake' test double --- .../mithril-dmq-node/src/test/double/mod.rs | 2 + .../src/test/double/publisher.rs | 61 +++++++++++++++++++ internal/mithril-dmq-node/src/test/mod.rs | 2 + 3 files changed, 65 insertions(+) create mode 100644 internal/mithril-dmq-node/src/test/double/publisher.rs diff --git a/internal/mithril-dmq-node/src/test/double/mod.rs b/internal/mithril-dmq-node/src/test/double/mod.rs index 5d3d2b87672..fd20a88aa0d 100644 --- a/internal/mithril-dmq-node/src/test/double/mod.rs +++ b/internal/mithril-dmq-node/src/test/double/mod.rs @@ -3,5 +3,7 @@ //! Enable unit testing with controlled inputs and predictable behavior. mod consumer; +mod publisher; pub use consumer::*; +pub use publisher::*; diff --git a/internal/mithril-dmq-node/src/test/double/publisher.rs b/internal/mithril-dmq-node/src/test/double/publisher.rs new file mode 100644 index 00000000000..4960ca25320 --- /dev/null +++ b/internal/mithril-dmq-node/src/test/double/publisher.rs @@ -0,0 +1,61 @@ +use std::{collections::VecDeque, fmt::Debug, marker::PhantomData}; + +use tokio::sync::Mutex; + +use mithril_common::{crypto_helper::TryToBytes, StdResult}; + +use crate::DmqPublisher; + +/// A fake implementation of the [DmqPublisher] trait for testing purposes. +pub struct DmqPublisherFake { + results: Mutex>>, + phantom: PhantomData, +} + +impl DmqPublisherFake { + /// Creates a new `DmqPublisherFake` instance with the provided results. + pub fn new(results: Vec>) -> Self { + Self { + results: Mutex::new(VecDeque::from(results)), + phantom: PhantomData, + } + } +} + +#[async_trait::async_trait] +impl DmqPublisher for DmqPublisherFake { + async fn publish_message(&self, _message: M) -> StdResult<()> { + let mut results = self.results.lock().await; + + results + .pop_front() + .ok_or_else(|| anyhow::anyhow!("No more results available in DmqPublisherFake"))? + } +} + +#[cfg(test)] +mod tests { + use crate::test::payload::DmqMessageTestPayload; + + use super::*; + + #[tokio::test] + async fn publish_messages_success() { + let publisher = DmqPublisherFake::new(vec![Ok(()), Err(anyhow::anyhow!("Test error"))]); + + publisher + .publish_message(DmqMessageTestPayload::new(b"test-1")) + .await + .unwrap(); + } + + #[tokio::test] + async fn publish_messages_failure() { + let publisher = DmqPublisherFake::new(vec![Err(anyhow::anyhow!("Test error")), Ok(())]); + + publisher + .publish_message(DmqMessageTestPayload::new(b"test-1")) + .await + .expect_err("DmqPublisherFake should return an error"); + } +} diff --git a/internal/mithril-dmq-node/src/test/mod.rs b/internal/mithril-dmq-node/src/test/mod.rs index be8d671b1aa..a1d1c1930d9 100644 --- a/internal/mithril-dmq-node/src/test/mod.rs +++ b/internal/mithril-dmq-node/src/test/mod.rs @@ -5,4 +5,6 @@ //! This module provides in particular test doubles for the traits defined in this crate. pub mod double; + +#[cfg(test)] pub(crate) mod payload; From 113cf94a8f385caba40e367555a2ade6973f043b Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Jun 2025 15:01:51 +0200 Subject: [PATCH 16/38] refactor(signer): 'SignaturePublisherDmq' uses a 'DmqPublisher' --- .../src/services/signature_publisher/dmq.rs | 83 +++++++++++++++++++ .../src/services/signature_publisher/mod.rs | 4 +- .../services/signature_publisher/pallas.rs | 29 ------- 3 files changed, 85 insertions(+), 31 deletions(-) create mode 100644 mithril-signer/src/services/signature_publisher/dmq.rs delete mode 100644 mithril-signer/src/services/signature_publisher/pallas.rs diff --git a/mithril-signer/src/services/signature_publisher/dmq.rs b/mithril-signer/src/services/signature_publisher/dmq.rs new file mode 100644 index 00000000000..c27c1ab9376 --- /dev/null +++ b/mithril-signer/src/services/signature_publisher/dmq.rs @@ -0,0 +1,83 @@ +use std::sync::Arc; + +use anyhow::Context; +use async_trait::async_trait; + +use mithril_common::{ + entities::{ProtocolMessage, SignedEntityType, SingleSignature}, + messages::RegisterSignatureMessageDmq, + StdResult, +}; +use mithril_dmq_node::DmqPublisher; + +use super::SignaturePublisher; + +/// DMQ implementation of the [SignaturePublisher] trait. +pub struct SignaturePublisherDmq { + dmq_publisher: Arc>, +} + +impl SignaturePublisherDmq { + /// Creates a new instance of [SignaturePublisherDmq]. + pub fn new(dmq_publisher: Arc>) -> Self { + Self { dmq_publisher } + } +} + +#[async_trait] +impl SignaturePublisher for SignaturePublisherDmq { + async fn publish( + &self, + signed_entity_type: &SignedEntityType, + signature: &SingleSignature, + _protocol_message: &ProtocolMessage, + ) -> StdResult<()> { + let message = RegisterSignatureMessageDmq { + signature: signature.signature.to_owned(), + signed_entity_type: signed_entity_type.to_owned(), + }; + + self.dmq_publisher + .publish_message(message) + .await + .with_context(|| "Failed to publish DMQ message") + } +} + +#[cfg(test)] +mod tests { + use mithril_common::test_utils::fake_data; + use mithril_dmq_node::test::double::DmqPublisherFake; + + use super::*; + + #[tokio::test] + async fn publish_signature_success() { + let signed_entity_type = SignedEntityType::dummy(); + let signature = fake_data::single_signature(vec![1, 2, 3]); + let protocol_message = ProtocolMessage::default(); + let dmq_publisher = Arc::new(DmqPublisherFake::new(vec![Ok(())])); + let publisher = SignaturePublisherDmq::new(dmq_publisher); + + publisher + .publish(&signed_entity_type, &signature, &protocol_message) + .await + .unwrap(); + } + + #[tokio::test] + async fn publish_signature_failure() { + let signed_entity_type = SignedEntityType::dummy(); + let signature = fake_data::single_signature(vec![1, 2, 3]); + let protocol_message = ProtocolMessage::default(); + let dmq_publisher = Arc::new(DmqPublisherFake::new(vec![Err(anyhow::anyhow!( + "Test error" + ))])); + let publisher = SignaturePublisherDmq::new(dmq_publisher); + + publisher + .publish(&signed_entity_type, &signature, &protocol_message) + .await + .expect_err("SignaturePublisherDmq should return an error"); + } +} diff --git a/mithril-signer/src/services/signature_publisher/mod.rs b/mithril-signer/src/services/signature_publisher/mod.rs index 09a37635148..099749b5b22 100644 --- a/mithril-signer/src/services/signature_publisher/mod.rs +++ b/mithril-signer/src/services/signature_publisher/mod.rs @@ -1,12 +1,12 @@ mod delayer; +mod dmq; mod http; mod interface; mod noop; -mod pallas; mod retrier; pub use delayer::*; +pub use dmq::*; pub use interface::*; pub use noop::*; -pub use pallas::*; pub use retrier::*; diff --git a/mithril-signer/src/services/signature_publisher/pallas.rs b/mithril-signer/src/services/signature_publisher/pallas.rs deleted file mode 100644 index 820df837617..00000000000 --- a/mithril-signer/src/services/signature_publisher/pallas.rs +++ /dev/null @@ -1,29 +0,0 @@ -use anyhow::Context; -use async_trait::async_trait; - -use mithril_common::{ - entities::{ProtocolMessage, SignedEntityType, SingleSignature}, - messages::RegisterSignatureMessageDmq, - StdResult, -}; -use mithril_dmq_node::{DmqPublisher, DmqPublisherPallas}; - -use super::SignaturePublisher; - -#[async_trait] -impl SignaturePublisher for DmqPublisherPallas { - async fn publish( - &self, - signed_entity_type: &SignedEntityType, - signature: &SingleSignature, - _protocol_message: &ProtocolMessage, - ) -> StdResult<()> { - let message = RegisterSignatureMessageDmq { - signature: signature.signature.to_owned(), - signed_entity_type: signed_entity_type.to_owned(), - }; - self.publish_message(message) - .await - .with_context(|| "Failed to publish DMQ message") - } -} From dbdfe613c7ffce14f99738c3c506a70226984ff1 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Jun 2025 15:25:49 +0200 Subject: [PATCH 17/38] refactor(aggregator): 'SignatureConsumerDmq' uses a 'DmqConsumer' --- .../src/services/signature_consumer/dmq.rs | 104 ++++++++++++++++++ .../src/services/signature_consumer/mod.rs | 4 +- .../src/services/signature_consumer/pallas.rs | 39 ------- 3 files changed, 106 insertions(+), 41 deletions(-) create mode 100644 mithril-aggregator/src/services/signature_consumer/dmq.rs delete mode 100644 mithril-aggregator/src/services/signature_consumer/pallas.rs diff --git a/mithril-aggregator/src/services/signature_consumer/dmq.rs b/mithril-aggregator/src/services/signature_consumer/dmq.rs new file mode 100644 index 00000000000..a5fead2e92e --- /dev/null +++ b/mithril-aggregator/src/services/signature_consumer/dmq.rs @@ -0,0 +1,104 @@ +use std::sync::Arc; + +use anyhow::Context; +use async_trait::async_trait; + +use mithril_common::{ + entities::{SignedEntityType, SingleSignature}, + messages::RegisterSignatureMessageDmq, + StdResult, +}; + +use mithril_dmq_node::DmqConsumer; + +use super::SignatureConsumer; + +/// DMQ implementation of the [SignatureConsumer] trait. +pub struct SignatureConsumerDmq { + dmq_consumer: Arc>, +} + +impl SignatureConsumerDmq { + /// Creates a new instance of [SignatureConsumerDmq]. + pub fn new(dmq_consumer: Arc>) -> Self { + Self { dmq_consumer } + } +} + +#[async_trait] +impl SignatureConsumer for SignatureConsumerDmq { + async fn get_signatures(&self) -> StdResult> { + self.dmq_consumer + .consume_messages() + .await + .map(|messages| { + messages + .into_iter() + .map(|(message, party_id)| { + let signature = message.signature; + let won_indexes = signature.indexes.clone(); + let single_signature = + SingleSignature::new(party_id, signature, won_indexes); + let signed_entity_type = message.signed_entity_type; + + (single_signature, signed_entity_type) + }) + .collect() + }) + .with_context(|| "Failed to get signatures from DMQ") + } + + fn get_origin_tag(&self) -> String { + "DMQ".to_string() + } +} + +#[cfg(test)] +mod tests { + use mithril_common::{crypto_helper::ProtocolSingleSignature, test_utils::fake_keys}; + use mithril_dmq_node::test::double::DmqConsumerFake; + + use super::*; + + #[tokio::test] + async fn get_signatures_success() { + let signed_entity_type = SignedEntityType::dummy(); + let single_signature: ProtocolSingleSignature = + fake_keys::single_signature()[0].try_into().unwrap(); + let dmq_consumer = Arc::new(DmqConsumerFake::new(vec![Ok(vec![( + RegisterSignatureMessageDmq { + signature: single_signature.clone(), + signed_entity_type: signed_entity_type.to_owned(), + }, + "pool-id-1".to_string(), + )])])); + let consumer = SignatureConsumerDmq::new(dmq_consumer); + + let signatures = consumer.get_signatures().await.unwrap(); + + assert_eq!( + vec![( + SingleSignature::new( + "pool-id-1".to_string(), + single_signature.clone(), + single_signature.indexes.clone() + ), + signed_entity_type + )], + signatures + ); + } + + #[tokio::test] + async fn get_signatures_failure() { + let dmq_consumer = Arc::new(DmqConsumerFake::new(vec![Err(anyhow::anyhow!( + "Test error" + ))])); + let consumer = SignatureConsumerDmq::new(dmq_consumer); + + consumer + .get_signatures() + .await + .expect_err("SignatureConsumerDmq should return an error"); + } +} diff --git a/mithril-aggregator/src/services/signature_consumer/mod.rs b/mithril-aggregator/src/services/signature_consumer/mod.rs index 7ed25404854..3b5068680a0 100644 --- a/mithril-aggregator/src/services/signature_consumer/mod.rs +++ b/mithril-aggregator/src/services/signature_consumer/mod.rs @@ -1,9 +1,9 @@ +mod dmq; mod fake; mod interface; mod noop; -mod pallas; +pub use dmq::*; pub use fake::*; pub use interface::*; pub use noop::*; -pub use pallas::*; diff --git a/mithril-aggregator/src/services/signature_consumer/pallas.rs b/mithril-aggregator/src/services/signature_consumer/pallas.rs deleted file mode 100644 index ceb2d2f141e..00000000000 --- a/mithril-aggregator/src/services/signature_consumer/pallas.rs +++ /dev/null @@ -1,39 +0,0 @@ -use anyhow::Context; -use async_trait::async_trait; - -use mithril_common::{ - entities::{SignedEntityType, SingleSignature}, - messages::RegisterSignatureMessageDmq, - StdResult, -}; - -use mithril_dmq_node::{DmqConsumer, DmqConsumerPallas}; - -use super::SignatureConsumer; - -#[async_trait] -impl SignatureConsumer for DmqConsumerPallas { - async fn get_signatures(&self) -> StdResult> { - self.consume_messages() - .await - .map(|messages| { - messages - .into_iter() - .map(|(message, party_id)| { - let signature = message.signature; - let won_indexes = signature.indexes.clone(); - let single_signature = - SingleSignature::new(party_id, signature, won_indexes); - let signed_entity_type = message.signed_entity_type; - - (single_signature, signed_entity_type) - }) - .collect() - }) - .with_context(|| "Failed to get signatures from DMQ") - } - - fn get_origin_tag(&self) -> String { - "DMQ".to_string() - } -} From 036f1341c4de44faed684fd6309c6e71f50c4659 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Jun 2025 17:28:52 +0200 Subject: [PATCH 18/38] feat(dmq): cache 'DmqClient' in 'DmqConsumer' --- internal/mithril-dmq-node/Cargo.toml | 4 +- .../mithril-dmq-node/src/consumer/pallas.rs | 133 ++++++++++++++++-- 2 files changed, 123 insertions(+), 14 deletions(-) diff --git a/internal/mithril-dmq-node/Cargo.toml b/internal/mithril-dmq-node/Cargo.toml index 7f02687e442..ef61d259ec9 100644 --- a/internal/mithril-dmq-node/Cargo.toml +++ b/internal/mithril-dmq-node/Cargo.toml @@ -21,10 +21,10 @@ mithril-cardano-node-chain = { path = "../cardano-node/mithril-cardano-node-chai mithril-common = { path = "../../mithril-common" } pallas-network = { git = "https://github.com/txpipe/pallas.git", branch = "main" } slog = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["sync"] } [dev-dependencies] mithril-common = { path = "../../mithril-common", features = ["test_tools"] } mockall = { workspace = true } slog-async = { workspace = true } -slog-term = { workspace = true } \ No newline at end of file +slog-term = { workspace = true } diff --git a/internal/mithril-dmq-node/src/consumer/pallas.rs b/internal/mithril-dmq-node/src/consumer/pallas.rs index 005aae69a48..15ae91a7c1d 100644 --- a/internal/mithril-dmq-node/src/consumer/pallas.rs +++ b/internal/mithril-dmq-node/src/consumer/pallas.rs @@ -3,6 +3,7 @@ use std::{fmt::Debug, marker::PhantomData, path::PathBuf}; use anyhow::{anyhow, Context}; use pallas_network::facades::DmqClient; use slog::{debug, error, Logger}; +use tokio::sync::{Mutex, MutexGuard}; use mithril_common::{ crypto_helper::{OpCert, TryFromBytes}, @@ -19,6 +20,7 @@ use crate::DmqConsumer; pub struct DmqConsumerPallas { socket: PathBuf, network: CardanoNetwork, + client: Mutex>, logger: Logger, phantom: PhantomData, } @@ -29,6 +31,7 @@ impl DmqConsumerPallas { Self { socket, network, + client: Mutex::new(None), logger: logger.new_with_component_name::(), phantom: PhantomData, } @@ -36,19 +39,64 @@ impl DmqConsumerPallas { /// Creates and returns a new `DmqClient` connected to the specified socket. async fn new_client(&self) -> StdResult { - let magic = self.network.code(); - DmqClient::connect(&self.socket, magic) + debug!( + self.logger, + "Create new DMQ client"; + "socket" => ?self.socket, + "network" => ?self.network + ); + DmqClient::connect(&self.socket, self.network.code()) .await .map_err(|err| anyhow!(err)) .with_context(|| "PallasChainReader failed to create a new client") } -} -#[async_trait::async_trait] -impl DmqConsumer for DmqConsumerPallas { - async fn consume_messages(&self) -> StdResult> { + /// Gets the cached `DmqClient`, creating a new one if it does not exist. + async fn get_client(&self) -> StdResult>> { + { + // Run this in a separate block to avoid dead lock on the Mutex + let client_lock = self.client.lock().await; + if client_lock.as_ref().is_some() { + return Ok(client_lock); + } + } + + let mut client_lock = self.client.lock().await; + *client_lock = Some(self.new_client().await?); + + Ok(client_lock) + } + + /// Drops the current `DmqClient`, if it exists. + async fn drop_client(&self) -> StdResult<()> { + debug!( + self.logger, + "Drop exsiting DMQ client"; + "socket" => ?self.socket, + "network" => ?self.network + ); + let mut client_lock = self.client.lock().await; + if let Some(client) = client_lock.take() { + client.abort().await; + } + + Ok(()) + } + + #[cfg(test)] + /// Check if the client already exists (test only). + async fn has_client(&self) -> bool { + let client_lock = self.client.lock().await; + + client_lock.as_ref().is_some() + } + + async fn consume_messages_internal(&self) -> StdResult> { debug!(self.logger, "Waiting for messages from DMQ..."); - let mut client = self.new_client().await?; // TODO: add client cache + let mut client_guard = self.get_client().await?; + let client = client_guard + .as_mut() + .ok_or(anyhow!("DMQ client does not exist"))?; client .msg_notification() .send_request_messages_blocking() @@ -78,13 +126,28 @@ impl DmqConsumer for DmqConsumerPallas } } -#[cfg(test)] +#[async_trait::async_trait] +impl DmqConsumer for DmqConsumerPallas { + async fn consume_messages(&self) -> StdResult> { + let messages = self.consume_messages_internal().await; + if messages.is_err() { + self.drop_client().await?; + } + + messages + } +} + +#[cfg(all(test, unix))] mod tests { use std::{fs, future, time::Duration, vec}; use mithril_common::{crypto_helper::TryToBytes, current_function, test_utils::TempDir}; - use pallas_network::miniprotocols::{localmsgnotification, localmsgsubmission::DmqMsg}; + use pallas_network::{ + facades::DmqServer, + miniprotocols::{localmsgnotification, localmsgsubmission::DmqMsg}, + }; use tokio::{net::UnixListener, task::JoinHandle, time::sleep}; use crate::{test::payload::DmqMessageTestPayload, test_tools::TestLogger}; @@ -135,7 +198,10 @@ mod tests { ] } - fn setup_dmq_server(socket_path: PathBuf, reply_messages: Vec) -> JoinHandle<()> { + fn setup_dmq_server( + socket_path: PathBuf, + reply_messages: Vec, + ) -> JoinHandle { tokio::spawn({ async move { // server setup @@ -169,6 +235,8 @@ mod tests { // server waits if no message available future::pending().await } + + server } }) } @@ -188,7 +256,8 @@ mod tests { consumer.consume_messages().await.unwrap() }); - let (_, messages) = tokio::join!(server, client); + let (_, client_res) = tokio::join!(server, client); + let messages = client_res.unwrap(); assert_eq!( vec![ @@ -201,7 +270,7 @@ mod tests { "pool17sln0evyk5tfj6zh2qrlk9vttgy6264sfe2fkec5mheasnlx3yd".to_string() ), ], - messages.unwrap() + messages ); } @@ -228,4 +297,44 @@ mod tests { result.expect_err("Should have timed out"); } + + #[tokio::test] + async fn pallas_dmq_consumer_client_is_dropped_when_returning_error() { + let socket_path = create_temp_dir(current_function!()).join("node.socket"); + let reply_messages = fake_msgs(); + let server = setup_dmq_server(socket_path.clone(), reply_messages); + let client = tokio::spawn(async move { + let consumer = DmqConsumerPallas::::new( + socket_path, + CardanoNetwork::TestNet(0), + TestLogger::stdout(), + ); + + consumer.consume_messages().await.unwrap(); + + consumer + }); + + let (server_res, client_res) = tokio::join!(server, client); + let consumer = client_res.unwrap(); + let server = server_res.unwrap(); + server.abort().await; + + let client = tokio::spawn(async move { + assert!(consumer.has_client().await, "Client should exist"); + + consumer + .consume_messages() + .await + .expect_err("Consuming messages should fail"); + + assert!( + !consumer.has_client().await, + "Client should have been dropped after error" + ); + + consumer + }); + client.await.unwrap(); + } } From ea6d7d6a6270ea89d6eed6d7fcb3c60d25349b0d Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 24 Jun 2025 12:09:45 +0200 Subject: [PATCH 19/38] feat(dmq): sign DMQ messages with 'KesSigner' --- internal/mithril-dmq-node/src/message.rs | 57 ++++++++++++------- .../mithril-dmq-node/src/publisher/pallas.rs | 32 ++++++++++- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/internal/mithril-dmq-node/src/message.rs b/internal/mithril-dmq-node/src/message.rs index 8d94dfba143..a8656046c86 100644 --- a/internal/mithril-dmq-node/src/message.rs +++ b/internal/mithril-dmq-node/src/message.rs @@ -5,29 +5,42 @@ use blake2::{digest::consts::U64, Blake2b, Digest}; use pallas_network::miniprotocols::localmsgsubmission::DmqMsg; use mithril_cardano_node_chain::chain_observer::ChainObserver; -use mithril_common::StdResult; +use mithril_common::{ + crypto_helper::{KesSigner, TryToBytes}, + StdResult, +}; /// The TTL (Time To Live) for DMQ messages in blocks. const DMQ_MESSAGE_TTL_IN_BLOCKS: u16 = 100; /// A builder for creating DMQ messages. pub struct DmqMessageBuilder { + kes_signer: Arc, chain_observer: Arc, ttl_blocks: u16, } impl DmqMessageBuilder { /// Creates a new instance of `DmqMessageBuilder`. - pub fn new(chain_observer: Arc, ttl_blocks: u16) -> Self { + pub fn new( + kes_signer: Arc, + chain_observer: Arc, + ttl_blocks: u16, + ) -> Self { Self { + kes_signer, chain_observer, ttl_blocks, } } /// Creates a new instance of `DmqMessageBuilder` with default TTL. - pub fn new_with_default_ttl(chain_observer: Arc) -> Self { + pub fn new_with_default_ttl( + kes_signer: Arc, + chain_observer: Arc, + ) -> Self { Self { + kes_signer, chain_observer, ttl_blocks: DMQ_MESSAGE_TTL_IN_BLOCKS, } @@ -58,15 +71,17 @@ impl DmqMessageBuilder { let block_number = (*block_number) .try_into() .map_err(|_| anyhow!("Failed to convert block number to u32"))?; - let kes_signature = vec![]; // TO DO: create a KES signature - let operational_certificate = vec![]; // TO DO: create an operational certificate + let (kes_signature, operational_certificate) = self + .kes_signer + .sign(message_bytes, block_number) + .with_context(|| "Failed to KES sign message while building DMQ message")?; let mut dmq_message = DmqMsg { msg_id: vec![], msg_body: message_bytes.to_vec(), block_number, ttl: self.ttl_blocks, - kes_signature, - operational_certificate, + kes_signature: kes_signature.to_bytes_vec()?, + operational_certificate: operational_certificate.to_bytes_vec()?, }; dmq_message.msg_id = compute_msg_id(&dmq_message); @@ -78,7 +93,7 @@ impl DmqMessageBuilder { mod tests { use mithril_cardano_node_chain::test::double::FakeChainObserver; use mithril_common::{ - crypto_helper::TryToBytes, + crypto_helper::{FakeKesSigner, TryToBytes}, entities::{BlockNumber, ChainPoint, TimePoint}, }; @@ -100,6 +115,11 @@ mod tests { #[tokio::test] async fn test_build_dmq_message() { + let (kes_signature, operational_certificate) = FakeKesSigner::dummy_signature(); + let kes_signer = Arc::new(FakeKesSigner::new(vec![Ok(( + kes_signature, + operational_certificate.clone(), + ))])); let chain_observer = Arc::new(FakeChainObserver::new(Some(TimePoint { chain_point: ChainPoint { block_number: BlockNumber(123), @@ -107,7 +127,7 @@ mod tests { }, ..TimePoint::dummy() }))); - let builder = DmqMessageBuilder::new(chain_observer, 100); + let builder = DmqMessageBuilder::new(kes_signer, chain_observer, 100); let message = test_utils::TestMessage { content: b"test".to_vec(), }; @@ -117,21 +137,20 @@ mod tests { .await .unwrap(); + assert!(!dmq_message.msg_id.is_empty()); assert_eq!( DmqMsg { - msg_id: vec![ - 26, 113, 171, 177, 174, 241, 244, 241, 209, 92, 210, 7, 119, 105, 94, 133, 93, - 62, 82, 95, 91, 221, 146, 174, 201, 190, 140, 1, 217, 240, 228, 203, 14, 50, - 104, 59, 252, 216, 26, 84, 231, 142, 163, 140, 11, 95, 17, 234, 242, 39, 230, - 160, 194, 219, 128, 42, 53, 125, 218, 48, 209, 3, 210, 154 - ], - msg_body: vec![116, 101, 115, 116], + msg_id: vec![], + msg_body: b"test".to_vec(), block_number: 123, ttl: 100, - kes_signature: vec![], - operational_certificate: vec![], + kes_signature: kes_signature.to_bytes_vec().unwrap(), + operational_certificate: operational_certificate.to_bytes_vec().unwrap(), }, - dmq_message + DmqMsg { + msg_id: vec![], + ..dmq_message + } ); } } diff --git a/internal/mithril-dmq-node/src/publisher/pallas.rs b/internal/mithril-dmq-node/src/publisher/pallas.rs index 5eb16399f9c..3199bdc6aa3 100644 --- a/internal/mithril-dmq-node/src/publisher/pallas.rs +++ b/internal/mithril-dmq-node/src/publisher/pallas.rs @@ -92,7 +92,7 @@ mod tests { use tokio::{net::UnixListener, task::JoinHandle}; use mithril_cardano_node_chain::test::double::FakeChainObserver; - use mithril_common::{current_function, test_utils::TempDir}; + use mithril_common::{crypto_helper::FakeKesSigner, current_function, test_utils::TempDir}; use crate::{test::payload::DmqMessageTestPayload, test_tools::TestLogger}; @@ -148,7 +148,20 @@ mod tests { let publisher = DmqPublisherPallas::new( socket_path, CardanoNetwork::TestNet(0), - DmqMessageBuilder::new(Arc::new(FakeChainObserver::default()), 100), + DmqMessageBuilder::new( + { + let (kes_signature, operational_certificate) = + FakeKesSigner::dummy_signature(); + let kes_signer = FakeKesSigner::new(vec![Ok(( + kes_signature, + operational_certificate.clone(), + ))]); + + Arc::new(kes_signer) + }, + Arc::new(FakeChainObserver::default()), + 100, + ), TestLogger::stdout(), ); @@ -171,7 +184,20 @@ mod tests { let publisher = DmqPublisherPallas::new( socket_path, CardanoNetwork::TestNet(0), - DmqMessageBuilder::new(Arc::new(FakeChainObserver::default()), 100), + DmqMessageBuilder::new( + { + let (kes_signature, operational_certificate) = + FakeKesSigner::dummy_signature(); + let kes_signer = FakeKesSigner::new(vec![Ok(( + kes_signature, + operational_certificate.clone(), + ))]); + + Arc::new(kes_signer) + }, + Arc::new(FakeChainObserver::default()), + 100, + ), TestLogger::stdout(), ); From 4c8581eddf40f8e4b501bee8ff6e954069e74ebb Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 24 Jun 2025 17:27:57 +0200 Subject: [PATCH 20/38] refactor(common): rename fake KES signer module --- internal/mithril-dmq-node/src/message.rs | 6 +++--- internal/mithril-dmq-node/src/publisher/pallas.rs | 10 +++++----- mithril-common/src/crypto_helper/cardano/kes/mod.rs | 4 ++-- .../cardano/kes/{fake_signer.rs => signer_fake.rs} | 10 +++++----- mithril-common/src/crypto_helper/mod.rs | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) rename mithril-common/src/crypto_helper/cardano/kes/{fake_signer.rs => signer_fake.rs} (93%) diff --git a/internal/mithril-dmq-node/src/message.rs b/internal/mithril-dmq-node/src/message.rs index a8656046c86..2d88ab90b7e 100644 --- a/internal/mithril-dmq-node/src/message.rs +++ b/internal/mithril-dmq-node/src/message.rs @@ -93,7 +93,7 @@ impl DmqMessageBuilder { mod tests { use mithril_cardano_node_chain::test::double::FakeChainObserver; use mithril_common::{ - crypto_helper::{FakeKesSigner, TryToBytes}, + crypto_helper::{KesSignerFake, TryToBytes}, entities::{BlockNumber, ChainPoint, TimePoint}, }; @@ -115,8 +115,8 @@ mod tests { #[tokio::test] async fn test_build_dmq_message() { - let (kes_signature, operational_certificate) = FakeKesSigner::dummy_signature(); - let kes_signer = Arc::new(FakeKesSigner::new(vec![Ok(( + let (kes_signature, operational_certificate) = KesSignerFake::dummy_signature(); + let kes_signer = Arc::new(KesSignerFake::new(vec![Ok(( kes_signature, operational_certificate.clone(), ))])); diff --git a/internal/mithril-dmq-node/src/publisher/pallas.rs b/internal/mithril-dmq-node/src/publisher/pallas.rs index 3199bdc6aa3..372fa6f99b1 100644 --- a/internal/mithril-dmq-node/src/publisher/pallas.rs +++ b/internal/mithril-dmq-node/src/publisher/pallas.rs @@ -92,7 +92,7 @@ mod tests { use tokio::{net::UnixListener, task::JoinHandle}; use mithril_cardano_node_chain::test::double::FakeChainObserver; - use mithril_common::{crypto_helper::FakeKesSigner, current_function, test_utils::TempDir}; + use mithril_common::{crypto_helper::KesSignerFake, current_function, test_utils::TempDir}; use crate::{test::payload::DmqMessageTestPayload, test_tools::TestLogger}; @@ -151,8 +151,8 @@ mod tests { DmqMessageBuilder::new( { let (kes_signature, operational_certificate) = - FakeKesSigner::dummy_signature(); - let kes_signer = FakeKesSigner::new(vec![Ok(( + KesSignerFake::dummy_signature(); + let kes_signer = KesSignerFake::new(vec![Ok(( kes_signature, operational_certificate.clone(), ))]); @@ -187,8 +187,8 @@ mod tests { DmqMessageBuilder::new( { let (kes_signature, operational_certificate) = - FakeKesSigner::dummy_signature(); - let kes_signer = FakeKesSigner::new(vec![Ok(( + KesSignerFake::dummy_signature(); + let kes_signer = KesSignerFake::new(vec![Ok(( kes_signature, operational_certificate.clone(), ))]); diff --git a/mithril-common/src/crypto_helper/cardano/kes/mod.rs b/mithril-common/src/crypto_helper/cardano/kes/mod.rs index b8f0bcc2314..dc512d61107 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/mod.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/mod.rs @@ -1,6 +1,6 @@ mod error; -mod fake_signer; mod interface; +mod signer_fake; mod signer_with_key; #[cfg(test)] pub(crate) mod tests_setup; @@ -8,7 +8,7 @@ pub(crate) mod tests_setup; mod verifier_standard; pub use error::*; -pub use fake_signer::*; pub use interface::*; +pub use signer_fake::*; pub use signer_with_key::*; pub use verifier_standard::*; diff --git a/mithril-common/src/crypto_helper/cardano/kes/fake_signer.rs b/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs similarity index 93% rename from mithril-common/src/crypto_helper/cardano/kes/fake_signer.rs rename to mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs index 2a930045806..1b4be18ef22 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/fake_signer.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs @@ -14,12 +14,12 @@ use crate::{ type KesSignatureResult = StdResult<(Sum6KesSig, OpCert)>; /// Fake KES Signer implementation. -pub struct FakeKesSigner { +pub struct KesSignerFake { results: Mutex>, } -impl FakeKesSigner { - /// Creates a new `FakeSignatureConsumer` instance. +impl KesSignerFake { + /// Creates a new `KesSignerFake` instance. pub fn new(results: Vec) -> Self { Self { results: Mutex::new(results.into()), @@ -50,7 +50,7 @@ impl FakeKesSigner { } } -impl KesSigner for FakeKesSigner { +impl KesSigner for KesSignerFake { fn sign(&self, _message: &[u8], _kes_period: KESPeriod) -> KesSignatureResult { let mut results = self.results.lock().unwrap(); @@ -76,7 +76,7 @@ mod tests { let (kes_signature, op_cert) = kes_signer .sign(message, kes_signing_period) .expect("Signing should not fail"); - let fake_kes_signer = FakeKesSigner::new(vec![ + let fake_kes_signer = KesSignerFake::new(vec![ Ok((kes_signature, op_cert.clone())), Err(anyhow::anyhow!("Fake error")), ]); diff --git a/mithril-common/src/crypto_helper/mod.rs b/mithril-common/src/crypto_helper/mod.rs index b6dc97bf9cb..9d1c8057098 100644 --- a/mithril-common/src/crypto_helper/mod.rs +++ b/mithril-common/src/crypto_helper/mod.rs @@ -16,8 +16,8 @@ cfg_test_tools! { pub use cardano::ColdKeyGenerator; pub use cardano::{ - KesPeriod, KesSigner, KesSignerStandard, KesVerifier, KesVerifierStandard, KesVerifyError, - OpCert, ProtocolInitializerErrorWrapper, ProtocolRegistrationErrorWrapper, + KesPeriod, KesSigner, KesSignerFake, KesSignerStandard, KesVerifier, KesVerifierStandard, + KesVerifyError, OpCert, ProtocolInitializerErrorWrapper, ProtocolRegistrationErrorWrapper, SerDeShelleyFileFormat, Sum6KesBytes, }; pub use codec::*; From 54a819e0a9149e44e9c27f9502f363d9bcbc1493 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 24 Jun 2025 17:58:04 +0200 Subject: [PATCH 21/38] refactor(common): 'test_setups' sub-module in 'kes' module --- .../src/crypto_helper/cardano/kes/mod.rs | 5 +-- .../crypto_helper/cardano/kes/signer_fake.rs | 39 ++++++++++++------- .../cardano/kes/signer_with_key.rs | 12 +++--- .../crypto_helper/cardano/kes/tests_setup.rs | 8 +++- .../cardano/kes/verifier_standard.rs | 16 ++++++-- .../cardano/key_certification.rs | 16 ++++++-- 6 files changed, 61 insertions(+), 35 deletions(-) diff --git a/mithril-common/src/crypto_helper/cardano/kes/mod.rs b/mithril-common/src/crypto_helper/cardano/kes/mod.rs index dc512d61107..9365d31d57c 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/mod.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/mod.rs @@ -2,13 +2,12 @@ mod error; mod interface; mod signer_fake; mod signer_with_key; -#[cfg(test)] -pub(crate) mod tests_setup; - +pub mod tests_setup; mod verifier_standard; pub use error::*; pub use interface::*; pub use signer_fake::*; pub use signer_with_key::*; +pub use tests_setup::*; pub use verifier_standard::*; diff --git a/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs b/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs index 1b4be18ef22..ba0613b327c 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs @@ -5,8 +5,11 @@ use std::sync::Mutex; use crate::{ crypto_helper::{ - cardano::{create_kes_cryptographic_material, KesSignerStandard}, - KESPeriod, KesSigner, OpCert, + cardano::{ + create_kes_cryptographic_material, KesCryptographicMaterialForTest, + KesPartyIndexForTest, KesSignerStandard, + }, + KesPeriod, KesSigner, OpCert, }, StdResult, }; @@ -28,12 +31,15 @@ impl KesSignerFake { /// Returns a dummy signature result that is always successful. pub fn dummy_signature() -> (Sum6KesSig, OpCert) { - let (_party_id, operational_certificate_file, kes_secret_key_file) = - create_kes_cryptographic_material( - 1, - 0 as KESPeriod, - "fake_kes_signer_returns_signature_batches_in_expected_order", - ); + let KesCryptographicMaterialForTest { + party_id: _, + operational_certificate_file, + kes_secret_key_file, + } = create_kes_cryptographic_material( + 1 as KesPartyIndexForTest, + 0 as KesPeriod, + "fake_kes_signer_returns_signature_batches_in_expected_order", + ); let message = b"Test message for KES signing"; let kes_signer = KesSignerStandard::new(kes_secret_key_file, operational_certificate_file); let kes_signing_period = 1; @@ -51,7 +57,7 @@ impl KesSignerFake { } impl KesSigner for KesSignerFake { - fn sign(&self, _message: &[u8], _kes_period: KESPeriod) -> KesSignatureResult { + fn sign(&self, _message: &[u8], _kes_period: KesPeriod) -> KesSignatureResult { let mut results = self.results.lock().unwrap(); results.pop_front().unwrap() @@ -64,12 +70,15 @@ mod tests { #[test] fn fake_kes_signer_returns_signature_batches_in_expected_order() { - let (_party_id, operational_certificate_file, kes_secret_key_file) = - create_kes_cryptographic_material( - 1, - 0 as KESPeriod, - "fake_kes_signer_returns_signature_batches_in_expected_order", - ); + let KesCryptographicMaterialForTest { + party_id: _, + operational_certificate_file, + kes_secret_key_file, + } = create_kes_cryptographic_material( + 1 as KesPartyIndexForTest, + 0 as KesPeriod, + "fake_kes_signer_returns_signature_batches_in_expected_order", + ); let message = b"Test message for KES signing"; let kes_signer = KesSignerStandard::new(kes_secret_key_file, operational_certificate_file); let kes_signing_period = 1; diff --git a/mithril-common/src/crypto_helper/cardano/kes/signer_with_key.rs b/mithril-common/src/crypto_helper/cardano/kes/signer_with_key.rs index 07fdd577df5..45b59505826 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/signer_with_key.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/signer_with_key.rs @@ -66,12 +66,10 @@ mod tests { use super::*; use crate::crypto_helper::cardano::{ - kes::tests_setup::create_kes_cryptographic_material, - tests_setup::KesCryptographicMaterialForTest, KesVerifier, KesVerifierStandard, + kes::tests_setup::{create_kes_cryptographic_material, KesCryptographicMaterialForTest}, + KesPartyIndexForTest, KesVerifier, KesVerifierStandard, }; - type PartyIndex = u64; - #[test] fn create_valid_signature_for_message() { let KesCryptographicMaterialForTest { @@ -79,7 +77,7 @@ mod tests { operational_certificate_file, kes_secret_key_file, } = create_kes_cryptographic_material( - 1 as PartyIndex, + 1 as KesPartyIndexForTest, 0 as KesPeriod, "create_valid_signature_for_message", ); @@ -103,7 +101,7 @@ mod tests { operational_certificate_file, kes_secret_key_file, } = create_kes_cryptographic_material( - 1 as PartyIndex, + 1 as KesPartyIndexForTest, 0 as KesPeriod, "create_invalid_signature_for_different_message", ); @@ -133,7 +131,7 @@ mod tests { operational_certificate_file, kes_secret_key_file, } = create_kes_cryptographic_material( - 1 as PartyIndex, + 1 as KesPartyIndexForTest, kes_period_start, "create_invalid_signature_for_invalid_kes_period", ); diff --git a/mithril-common/src/crypto_helper/cardano/kes/tests_setup.rs b/mithril-common/src/crypto_helper/cardano/kes/tests_setup.rs index 20473797f79..e64db4d8bd9 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/tests_setup.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/tests_setup.rs @@ -6,16 +6,20 @@ use kes_summed_ed25519::traits::KesSk; use crate::crypto_helper::{cardano::ColdKeyGenerator, OpCert}; use crate::crypto_helper::{KesPeriod, ProtocolPartyId, SerDeShelleyFileFormat, Sum6KesBytes}; +/// A type alias for the party index used in KES cryptographic material. +pub type KesPartyIndexForTest = u64; + /// A struct to hold KES cryptographic material for testing purposes. pub(crate) struct KesCryptographicMaterialForTest { + #[allow(dead_code)] pub party_id: ProtocolPartyId, pub operational_certificate_file: PathBuf, pub kes_secret_key_file: PathBuf, } /// Create KES cryptographic material for testing purposes. -pub(crate) fn create_kes_cryptographic_material( - party_idx: u64, +pub fn create_kes_cryptographic_material( + party_idx: KesPartyIndexForTest, kes_period: KesPeriod, test_directory: &str, ) -> KesCryptographicMaterialForTest { diff --git a/mithril-common/src/crypto_helper/cardano/kes/verifier_standard.rs b/mithril-common/src/crypto_helper/cardano/kes/verifier_standard.rs index ecc4f55ee62..f9d05d196a2 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/verifier_standard.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/verifier_standard.rs @@ -48,8 +48,8 @@ impl KesVerifier for KesVerifierStandard { #[cfg(test)] mod tests { use crate::crypto_helper::cardano::{ - kes::tests_setup::create_kes_cryptographic_material, - tests_setup::KesCryptographicMaterialForTest, KesSigner, KesSignerStandard, + kes::tests_setup::{create_kes_cryptographic_material, KesCryptographicMaterialForTest}, + KesPartyIndexForTest, KesSigner, KesSignerStandard, }; use super::*; @@ -60,7 +60,11 @@ mod tests { party_id: _, operational_certificate_file, kes_secret_key_file, - } = create_kes_cryptographic_material(1, 0 as KesPeriod, "verify_valid_signature_succeeds"); + } = create_kes_cryptographic_material( + 1 as KesPartyIndexForTest, + 0 as KesPeriod, + "verify_valid_signature_succeeds", + ); let message = b"Test message for KES signing"; let kes_signer = KesSignerStandard::new(kes_secret_key_file, operational_certificate_file); let kes_signing_period = 1; @@ -79,7 +83,11 @@ mod tests { party_id: _, operational_certificate_file, kes_secret_key_file, - } = create_kes_cryptographic_material(1, 0 as KesPeriod, "verify_invalid_signature_fails"); + } = create_kes_cryptographic_material( + 1 as KesPartyIndexForTest, + 0 as KesPeriod, + "verify_invalid_signature_fails", + ); let message = b"Test message for KES signing"; let kes_signer = KesSignerStandard::new(kes_secret_key_file, operational_certificate_file); let kes_signing_period = 1; diff --git a/mithril-common/src/crypto_helper/cardano/key_certification.rs b/mithril-common/src/crypto_helper/cardano/key_certification.rs index aafe18c0f9a..e173ac5147d 100644 --- a/mithril-common/src/crypto_helper/cardano/key_certification.rs +++ b/mithril-common/src/crypto_helper/cardano/key_certification.rs @@ -300,9 +300,9 @@ impl KeyRegWrapper { #[cfg(test)] mod test { use super::*; - use crate::crypto_helper::cardano::tests_setup::KesCryptographicMaterialForTest; use crate::crypto_helper::cardano::{ - tests_setup::create_kes_cryptographic_material, KesSignerStandard, + kes::tests_setup::{create_kes_cryptographic_material, KesCryptographicMaterialForTest}, + KesPartyIndexForTest, KesSignerStandard, }; use crate::crypto_helper::{OpCert, SerDeShelleyFileFormat}; @@ -321,12 +321,20 @@ mod test { party_id: party_id_1, operational_certificate_file: operational_certificate_file_1, kes_secret_key_file: kes_secret_key_file_1, - } = create_kes_cryptographic_material(1, 0 as KesPeriod, "test_vector_key_reg"); + } = create_kes_cryptographic_material( + 1 as KesPartyIndexForTest, + 0 as KesPeriod, + "test_vector_key_reg", + ); let KesCryptographicMaterialForTest { party_id: party_id_2, operational_certificate_file: operational_certificate_file_2, kes_secret_key_file: kes_secret_key_file_2, - } = create_kes_cryptographic_material(2, 0 as KesPeriod, "test_vector_key_reg"); + } = create_kes_cryptographic_material( + 2 as KesPartyIndexForTest, + 0 as KesPeriod, + "test_vector_key_reg", + ); let mut key_reg = KeyRegWrapper::init(&vec![(party_id_1, 10), (party_id_2, 3)]); From 55827c1e0dcb678423904651496cb8a829b837e7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 26 Jun 2025 14:53:27 +0200 Subject: [PATCH 22/38] feat(signer): wire 'SignaturePublisherDmq' in DI --- mithril-signer/src/configuration.rs | 10 +++- .../src/dependency_injection/builder.rs | 57 ++++++++++++++++--- .../src/dependency_injection/containers.rs | 4 ++ mithril-signer/src/runtime/runner.rs | 28 ++------- .../test_extensions/state_machine_tester.rs | 6 ++ 5 files changed, 70 insertions(+), 35 deletions(-) diff --git a/mithril-signer/src/configuration.rs b/mithril-signer/src/configuration.rs index efaa211b1c1..5f1f9698e65 100644 --- a/mithril-signer/src/configuration.rs +++ b/mithril-signer/src/configuration.rs @@ -35,11 +35,14 @@ pub struct Configuration { #[example = "`cardano-cli`"] pub cardano_cli_path: PathBuf, - /// Path of the socket used by the Cardano CLI tool - /// to communicate with the Cardano node - #[example = "`/tmp/cardano.sock`"] + /// Path of the socket opened by the Cardano node + #[example = "`/ipc/node.socket`"] pub cardano_node_socket_path: PathBuf, + /// Path of the socket opened by the DMQ node + #[example = "`/ipc/dmq.socket`"] + pub dmq_node_socket_path: Option, + /// Cardano network #[example = "`testnet` or `mainnet` or `devnet`"] pub network: String, @@ -150,6 +153,7 @@ impl Configuration { relay_endpoint: None, cardano_cli_path: PathBuf::new(), cardano_node_socket_path: PathBuf::new(), + dmq_node_socket_path: None, db_directory: PathBuf::new(), network: "devnet".to_string(), network_magic: Some(42), diff --git a/mithril-signer/src/dependency_injection/builder.rs b/mithril-signer/src/dependency_injection/builder.rs index cb232dfaace..d71d5fb0ef9 100644 --- a/mithril-signer/src/dependency_injection/builder.rs +++ b/mithril-signer/src/dependency_injection/builder.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, Context}; +use mithril_dmq_node::{DmqMessageBuilder, DmqPublisherPallas}; use slog::Logger; use tokio::sync::{Mutex, RwLock}; @@ -19,14 +20,16 @@ use mithril_cardano_node_internal_database::{ signable_builder::{CardanoDatabaseSignableBuilder, CardanoImmutableFilesFullSignableBuilder}, ImmutableFileObserver, ImmutableFileSystemObserver, }; -use mithril_common::api_version::APIVersionProvider; -use mithril_common::crypto_helper::{OpCert, ProtocolPartyId, SerDeShelleyFileFormat}; +use mithril_common::crypto_helper::{ + KesSigner, KesSignerStandard, OpCert, ProtocolPartyId, SerDeShelleyFileFormat, +}; use mithril_common::signable_builder::{ CardanoStakeDistributionSignableBuilder, CardanoTransactionsSignableBuilder, MithrilSignableBuilderService, MithrilStakeDistributionSignableBuilder, SignableBuilderServiceDependencies, }; use mithril_common::StdResult; +use mithril_common::{api_version::APIVersionProvider, messages::RegisterSignatureMessageDmq}; use mithril_era::{EraChecker, EraReader}; use mithril_signed_entity_lock::SignedEntityTypeLock; @@ -37,10 +40,6 @@ use mithril_persistence::database::repository::CardanoTransactionRepository; use mithril_persistence::database::{ApplicationNodeType, SqlMigration}; use mithril_persistence::sqlite::{ConnectionBuilder, SqliteConnection, SqliteConnectionPool}; -use crate::database::repository::{ - ProtocolInitializerRepository, SignedBeaconRepository, StakePoolStore, -}; -use crate::dependency_injection::SignerDependencyContainer; use crate::services::{ AggregatorHTTPClient, CardanoTransactionsImporter, CardanoTransactionsPreloaderActivationSigner, MithrilEpochService, MithrilSingleSigner, @@ -50,6 +49,11 @@ use crate::services::{ TransactionsImporterWithPruner, TransactionsImporterWithVacuum, }; use crate::store::MKTreeStoreSqlite; +use crate::{ + database::repository::{ProtocolInitializerRepository, SignedBeaconRepository, StakePoolStore}, + services::SignaturePublisher, +}; +use crate::{dependency_injection::SignerDependencyContainer, services::SignaturePublisherDmq}; use crate::{ Configuration, MetricsService, HTTP_REQUEST_TIMEOUT_DURATION, SQLITE_FILE, SQLITE_FILE_CARDANO_TRANSACTION, @@ -390,10 +394,46 @@ impl<'a> DependenciesBuilder<'a> { self.root_logger(), )); + let kes_signer = match ( + &self.config.kes_secret_key_path, + &self.config.operational_certificate_path, + ) { + (Some(kes_secret_key_path), Some(operational_certificate_path)) => { + Some(Arc::new(KesSignerStandard::new( + kes_secret_key_path.clone(), + operational_certificate_path.clone(), + )) as Arc) + } + (Some(_), None) | (None, Some(_)) => { + return Err(anyhow!( + "kes_secret_key and operational_certificate are both mandatory".to_string(), + )) + } + _ => None, + }; + + let cardano_network = &self.config.get_network()?; + let signature_publisher = { - // Temporary no-op publisher before a DMQ-based implementation is available. let first_publisher = SignaturePublisherRetrier::new( - Arc::new(SignaturePublisherNoop {}), + if let Some(dmq_node_socket_path) = &self.config.dmq_node_socket_path { + let dmq_message_builder = DmqMessageBuilder::new_with_default_ttl( + kes_signer + .clone() + .ok_or(anyhow!("A KES signer is mandatory to sign DMQ messages"))?, + chain_observer.clone(), + ); + Arc::new(SignaturePublisherDmq::new(Arc::new(DmqPublisherPallas::< + RegisterSignatureMessageDmq, + >::new( + dmq_node_socket_path.to_owned(), + *cardano_network, + dmq_message_builder, + self.root_logger(), + )))) as Arc + } else { + Arc::new(SignaturePublisherNoop) as Arc + }, SignaturePublishRetryPolicy::never(), ); @@ -442,6 +482,7 @@ impl<'a> DependenciesBuilder<'a> { upkeep_service, epoch_service, certifier, + kes_signer, }; Ok(services) diff --git a/mithril-signer/src/dependency_injection/containers.rs b/mithril-signer/src/dependency_injection/containers.rs index 5cf9b9eb8bf..0bf258e4802 100644 --- a/mithril-signer/src/dependency_injection/containers.rs +++ b/mithril-signer/src/dependency_injection/containers.rs @@ -1,3 +1,4 @@ +use mithril_common::crypto_helper::KesSigner; use std::sync::Arc; use tokio::sync::RwLock; @@ -81,4 +82,7 @@ pub struct SignerDependencyContainer { /// Certifier service pub certifier: Arc, + + /// Kes signer service + pub kes_signer: Option>, } diff --git a/mithril-signer/src/runtime/runner.rs b/mithril-signer/src/runtime/runner.rs index 286fd34400d..b22afce6b7b 100644 --- a/mithril-signer/src/runtime/runner.rs +++ b/mithril-signer/src/runtime/runner.rs @@ -1,14 +1,10 @@ -use std::sync::Arc; - use anyhow::Context; use async_trait::async_trait; use slog::{debug, warn, Logger}; use thiserror::Error; use tokio::sync::RwLockReadGuard; -use mithril_common::crypto_helper::{ - KesPeriod, KesSigner, KesSignerStandard, OpCert, ProtocolOpCert, SerDeShelleyFileFormat, -}; +use mithril_common::crypto_helper::{KesPeriod, OpCert, ProtocolOpCert, SerDeShelleyFileFormat}; use mithril_common::entities::{ Epoch, PartyId, ProtocolMessage, SignedEntityType, Signer, TimePoint, }; @@ -186,28 +182,10 @@ impl Runner for SignerRunner { ), None => None, }; - let kes_signer = match ( - &self.config.kes_secret_key_path, - &self.config.operational_certificate_path, - ) { - (Some(kes_secret_key_path), Some(operational_certificate_path)) => { - Some(Arc::new(KesSignerStandard::new( - kes_secret_key_path.clone(), - operational_certificate_path.clone(), - )) as Arc) - } - (Some(_), None) | (None, Some(_)) => { - return Err(RunnerError::NoValueError( - "kes_secret_key and operational_certificate are both mandatory".to_string(), - ) - .into()) - } - _ => None, - }; let protocol_initializer = MithrilProtocolInitializerBuilder::build( stake, &protocol_parameters, - kes_signer, + self.services.kes_signer.clone(), kes_period, )?; let signer = Signer::new( @@ -528,6 +506,7 @@ mod tests { aggregator_client.clone(), logger.clone(), )); + let kes_signer = None; SignerDependencyContainer { stake_store, @@ -547,6 +526,7 @@ mod tests { upkeep_service, epoch_service, certifier, + kes_signer, } } diff --git a/mithril-signer/tests/test_extensions/state_machine_tester.rs b/mithril-signer/tests/test_extensions/state_machine_tester.rs index 1c5ffe95718..7da1eac2234 100644 --- a/mithril-signer/tests/test_extensions/state_machine_tester.rs +++ b/mithril-signer/tests/test_extensions/state_machine_tester.rs @@ -27,6 +27,7 @@ use mithril_cardano_node_internal_database::{ }; use mithril_common::{ api_version::APIVersionProvider, + crypto_helper::{KesSigner, KesSignerStandard}, entities::{ BlockNumber, CardanoTransactionsSigningConfig, ChainPoint, Epoch, SignedEntityConfig, SignedEntityType, SignedEntityTypeDiscriminants, SignerWithStake, SlotNumber, SupportedEra, @@ -287,6 +288,10 @@ impl StateMachineTester { certificate_handler.clone(), logger.clone(), )); + let kes_signer = Some(Arc::new(KesSignerStandard::new( + config.kes_secret_key_path.clone().unwrap(), + config.operational_certificate_path.clone().unwrap(), + )) as Arc); let services = SignerDependencyContainer { certificate_handler: certificate_handler.clone(), @@ -306,6 +311,7 @@ impl StateMachineTester { upkeep_service, epoch_service, certifier, + kes_signer, }; // set up stake distribution chain_observer From c7d529c1b9b2c710c080d224939cde3a69557a26 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 26 Jun 2025 15:12:28 +0200 Subject: [PATCH 23/38] feat(aggregator): wire 'SignatureConsumerDmq' in DI --- mithril-aggregator/src/configuration.rs | 22 ++++++++++++++----- .../builder/enablers/misc.rs | 19 +++++++++++++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/mithril-aggregator/src/configuration.rs b/mithril-aggregator/src/configuration.rs index 76509df4c69..b925a1c14e4 100644 --- a/mithril-aggregator/src/configuration.rs +++ b/mithril-aggregator/src/configuration.rs @@ -61,12 +61,16 @@ pub trait ConfigurationSource { panic!("cardano_cli_path is not implemented."); } - /// Path of the socket used by the Cardano CLI tool - /// to communicate with the Cardano node + /// Path of the socket opened by the Cardano node fn cardano_node_socket_path(&self) -> PathBuf { panic!("cardano_node_socket_path is not implemented."); } + /// Path of the socket opened by the DMQ node + fn dmq_node_socket_path(&self) -> Option { + panic!("dmq_node_socket_path is not implemented."); + } + /// Cardano node version. /// /// **NOTE**: This cannot be verified for now (see [this @@ -390,11 +394,14 @@ pub struct ServeCommandConfiguration { #[example = "`cardano-cli`"] pub cardano_cli_path: PathBuf, - /// Path of the socket used by the Cardano CLI tool - /// to communicate with the Cardano node - #[example = "`/tmp/cardano.sock`"] + /// Path of the socket opened by the Cardano node + #[example = "`/ipc/node.socket`"] pub cardano_node_socket_path: PathBuf, + /// Path of the socket opened by the DMQ node + #[example = "`/ipc/dmq.socket`"] + pub dmq_node_socket_path: Option, + /// Cardano node version. /// /// **NOTE**: This cannot be verified for now (see [this @@ -628,6 +635,7 @@ impl ServeCommandConfiguration { environment: ExecutionEnvironment::Test, cardano_cli_path: PathBuf::new(), cardano_node_socket_path: PathBuf::new(), + dmq_node_socket_path: None, cardano_node_version: "0.0.1".to_string(), network_magic: Some(42), network: "devnet".to_string(), @@ -707,6 +715,10 @@ impl ConfigurationSource for ServeCommandConfiguration { self.cardano_node_socket_path.clone() } + fn dmq_node_socket_path(&self) -> Option { + self.dmq_node_socket_path.clone() + } + fn cardano_node_version(&self) -> String { self.cardano_node_version.clone() } diff --git a/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs b/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs index 1cf29aa7fec..a3da44b9144 100644 --- a/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs +++ b/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs @@ -7,6 +7,8 @@ use std::sync::Arc; use std::time::Duration; +use mithril_common::messages::RegisterSignatureMessageDmq; +use mithril_dmq_node::DmqConsumerPallas; use mithril_signed_entity_lock::SignedEntityTypeLock; use crate::database::repository::CertificateRepository; @@ -14,7 +16,8 @@ use crate::dependency_injection::{DependenciesBuilder, Result}; use crate::get_dependency; use crate::services::{ AggregatorClient, AggregatorHTTPClient, MessageService, MithrilMessageService, - SequentialSignatureProcessor, SignatureConsumer, SignatureConsumerNoop, SignatureProcessor, + SequentialSignatureProcessor, SignatureConsumer, SignatureConsumerDmq, SignatureConsumerNoop, + SignatureProcessor, }; impl DependenciesBuilder { async fn build_signed_entity_type_lock(&mut self) -> Result> { @@ -74,9 +77,19 @@ impl DependenciesBuilder { /// Builds a [SignatureConsumer] pub async fn build_signature_consumer(&mut self) -> Result> { - let signature_consumer = SignatureConsumerNoop; + let signature_consumer = + if let Some(dmq_node_socket_path) = self.configuration.dmq_node_socket_path() { + let dmq_consumer = Arc::new(DmqConsumerPallas::::new( + dmq_node_socket_path, + self.configuration.get_network()?, + self.root_logger(), + )); + Arc::new(SignatureConsumerDmq::new(dmq_consumer)) as Arc + } else { + Arc::new(SignatureConsumerNoop) as Arc + }; - Ok(Arc::new(signature_consumer)) + Ok(signature_consumer) } /// Builds a [SignatureProcessor] From dae0a7ffe3fe30cc96c20923d8cb6007866393a7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 26 Jun 2025 17:09:30 +0200 Subject: [PATCH 24/38] docs: better socket file path examples --- .../develop/nodes/mithril-aggregator.md | 86 +++++++++---------- .../manual/develop/nodes/mithril-signer.md | 2 +- .../develop/nodes/mithril-aggregator.md | 86 +++++++++---------- .../manual/develop/nodes/mithril-signer.md | 2 +- mithril-aggregator/config/dev.json | 2 +- mithril-aggregator/config/preview.json | 2 +- .../src/commands/genesis_command.rs | 5 +- 7 files changed, 92 insertions(+), 93 deletions(-) diff --git a/docs/website/root/manual/develop/nodes/mithril-aggregator.md b/docs/website/root/manual/develop/nodes/mithril-aggregator.md index a0297073009..574b006bdc6 100644 --- a/docs/website/root/manual/develop/nodes/mithril-aggregator.md +++ b/docs/website/root/manual/develop/nodes/mithril-aggregator.md @@ -495,21 +495,21 @@ The configuration parameters can be set in either of the following ways: Here is a list of the available parameters for the serve command: -| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | -| ------------------------------ | -------------------- | :------------------: | ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------ | -------------------------- | ----------------------------------------------------------------------------------------------------------------------- | :----------------: | -| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | -| `config_directory` | `--config-directory` | - | - | Directory of the configuration file | `./config` | - | - | -| `data_stores_directory` | - | - | `data_stores_directory` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | -| `db_directory` | `--db-directory` | - | `DB_DIRECTORY` | Directory of the **Cardano node** stores | `/db` | - | :heavy_check_mark: | -| `genesis_verification_key` | - | - | `GENESIS_VERIFICATION_KEY` | Genesis verification key | - | - | :heavy_check_mark: | -| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | -| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | -| `protocol_parameters` | - | - | `PROTOCOL_PARAMETERS__K`, `PROTOCOL_PARAMETERS__M`, and `PROTOCOL_PARAMETERS__PHI_F` | Mithril protocol parameters | - | `{ k: 5, m: 100, phi_f: 0.65 }` | :heavy_check_mark: | -| `run_mode` | `--run-mode` | `-r` | `RUN_MODE` | Runtime mode | `dev` | - | :heavy_check_mark: | -| `store_retention_limit` | - | - | `STORE_RETENTION_LIMIT` | Maximum number of records in stores. If not set, no limit is set. | - | - | - | -| `custom_origin_tag_white_list` | - | - | `CUSTOM_ORIGIN_TAG_WHITE_LIST` | Custom origin tag of client request added to the whitelist (comma separated list). | `EXPLORER,BENCHMARK,CI,NA` | `EXAMPLE` | - | -| `verbose` | `--verbose` | `-v` | `VERBOSE` | Verbosity level | - | Parsed from the number of occurrences: `-v` for `Warning`, `-vv` for `Info`, `-vvv` for `Debug` and `-vvvv` for `Trace` | :heavy_check_mark: | +| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | +| ------------------------------ | -------------------- | :------------------: | ------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------- | -------------------------- | ----------------------------------------------------------------------------------------------------------------------- | :----------------: | +| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | +| `config_directory` | `--config-directory` | - | - | Directory of the configuration file | `./config` | - | - | +| `data_stores_directory` | - | - | `data_stores_directory` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | +| `db_directory` | `--db-directory` | - | `DB_DIRECTORY` | Directory of the **Cardano node** stores | `/db` | - | :heavy_check_mark: | +| `genesis_verification_key` | - | - | `GENESIS_VERIFICATION_KEY` | Genesis verification key | - | - | :heavy_check_mark: | +| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | +| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | +| `protocol_parameters` | - | - | `PROTOCOL_PARAMETERS__K`, `PROTOCOL_PARAMETERS__M`, and `PROTOCOL_PARAMETERS__PHI_F` | Mithril protocol parameters | - | `{ k: 5, m: 100, phi_f: 0.65 }` | :heavy_check_mark: | +| `run_mode` | `--run-mode` | `-r` | `RUN_MODE` | Runtime mode | `dev` | - | :heavy_check_mark: | +| `store_retention_limit` | - | - | `STORE_RETENTION_LIMIT` | Maximum number of records in stores. If not set, no limit is set. | - | - | - | +| `custom_origin_tag_white_list` | - | - | `CUSTOM_ORIGIN_TAG_WHITE_LIST` | Custom origin tag of client request added to the whitelist (comma separated list). | `EXPLORER,BENCHMARK,CI,NA` | `EXAMPLE` | - | +| `verbose` | `--verbose` | `-v` | `VERBOSE` | Verbosity level | - | Parsed from the number of occurrences: `-v` for `Warning`, `-vv` for `Info`, `-vvv` for `Debug` and `-vvvv` for `Trace` | :heavy_check_mark: | `serve` command: @@ -545,40 +545,40 @@ Here is a list of the available parameters for the serve command: `genesis bootstrap` command: -| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | -| -------------------------- | ------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | -| `genesis_secret_key` | - | - | `GENESIS_SECRET_KEY` | Genesis secret key, :warning: for test only | - | - | :heavy_check_mark: | -| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | -| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | -| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | -| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | -| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | +| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | +| -------------------------- | ------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | +| `genesis_secret_key` | - | - | `GENESIS_SECRET_KEY` | Genesis secret key, :warning: for test only | - | - | :heavy_check_mark: | +| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | +| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | +| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | +| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | +| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | `genesis export` command: -| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | -| -------------------------- | ------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | -| `target_path` | `--target-path` | - | - | Path of the file to export the payload to. | - | - | :heavy_check_mark: | -| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | -| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | -| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | -| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | -| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | +| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | +| -------------------------- | ------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | +| `target_path` | `--target-path` | - | - | Path of the file to export the payload to. | - | - | :heavy_check_mark: | +| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | +| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | +| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | +| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | +| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | `genesis import` command: -| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | -| -------------------------- | ---------------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | -| `signed_payload_path` | `--signed-payload-path` | - | - | Path of the payload to import. | - | - | :heavy_check_mark: | -| `genesis_verification_key` | `--genesis-verification-key` | - | - | Genesis verification key | - | - | :heavy_check_mark: | -| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | -| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | -| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | -| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | -| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | +| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | +| -------------------------- | ---------------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | +| `signed_payload_path` | `--signed-payload-path` | - | - | Path of the payload to import. | - | - | :heavy_check_mark: | +| `genesis_verification_key` | `--genesis-verification-key` | - | - | Genesis verification key | - | - | :heavy_check_mark: | +| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | +| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | +| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | +| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | +| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | `genesis sign` command: diff --git a/docs/website/root/manual/develop/nodes/mithril-signer.md b/docs/website/root/manual/develop/nodes/mithril-signer.md index 1d2fcc15922..df6b0a08e15 100644 --- a/docs/website/root/manual/develop/nodes/mithril-signer.md +++ b/docs/website/root/manual/develop/nodes/mithril-signer.md @@ -236,7 +236,7 @@ Here is a list of the available parameters: | `verbose` | `--verbose` | `-v` | `VERBOSE` | Verbosity level | - | Parsed from the number of occurrences: `-v` for `Warning`, `-vv` for `Info`, `-vvv` for `Debug` and `-vvvv` for `Trace` | :heavy_check_mark: | | `run_mode` | `--run-mode` | `-r` | `RUN_MODE` | Runtime mode | `dev` | - | :heavy_check_mark: | | `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | | `db_directory` | `--db-directory` | - | `DB_DIRECTORY` | Directory to snapshot from the **Cardano node** | `/db` | - | :heavy_check_mark: | | `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | | `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | diff --git a/docs/website/versioned_docs/version-maintained/manual/develop/nodes/mithril-aggregator.md b/docs/website/versioned_docs/version-maintained/manual/develop/nodes/mithril-aggregator.md index a0297073009..574b006bdc6 100644 --- a/docs/website/versioned_docs/version-maintained/manual/develop/nodes/mithril-aggregator.md +++ b/docs/website/versioned_docs/version-maintained/manual/develop/nodes/mithril-aggregator.md @@ -495,21 +495,21 @@ The configuration parameters can be set in either of the following ways: Here is a list of the available parameters for the serve command: -| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | -| ------------------------------ | -------------------- | :------------------: | ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------ | -------------------------- | ----------------------------------------------------------------------------------------------------------------------- | :----------------: | -| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | -| `config_directory` | `--config-directory` | - | - | Directory of the configuration file | `./config` | - | - | -| `data_stores_directory` | - | - | `data_stores_directory` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | -| `db_directory` | `--db-directory` | - | `DB_DIRECTORY` | Directory of the **Cardano node** stores | `/db` | - | :heavy_check_mark: | -| `genesis_verification_key` | - | - | `GENESIS_VERIFICATION_KEY` | Genesis verification key | - | - | :heavy_check_mark: | -| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | -| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | -| `protocol_parameters` | - | - | `PROTOCOL_PARAMETERS__K`, `PROTOCOL_PARAMETERS__M`, and `PROTOCOL_PARAMETERS__PHI_F` | Mithril protocol parameters | - | `{ k: 5, m: 100, phi_f: 0.65 }` | :heavy_check_mark: | -| `run_mode` | `--run-mode` | `-r` | `RUN_MODE` | Runtime mode | `dev` | - | :heavy_check_mark: | -| `store_retention_limit` | - | - | `STORE_RETENTION_LIMIT` | Maximum number of records in stores. If not set, no limit is set. | - | - | - | -| `custom_origin_tag_white_list` | - | - | `CUSTOM_ORIGIN_TAG_WHITE_LIST` | Custom origin tag of client request added to the whitelist (comma separated list). | `EXPLORER,BENCHMARK,CI,NA` | `EXAMPLE` | - | -| `verbose` | `--verbose` | `-v` | `VERBOSE` | Verbosity level | - | Parsed from the number of occurrences: `-v` for `Warning`, `-vv` for `Info`, `-vvv` for `Debug` and `-vvvv` for `Trace` | :heavy_check_mark: | +| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | +| ------------------------------ | -------------------- | :------------------: | ------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------- | -------------------------- | ----------------------------------------------------------------------------------------------------------------------- | :----------------: | +| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | +| `config_directory` | `--config-directory` | - | - | Directory of the configuration file | `./config` | - | - | +| `data_stores_directory` | - | - | `data_stores_directory` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | +| `db_directory` | `--db-directory` | - | `DB_DIRECTORY` | Directory of the **Cardano node** stores | `/db` | - | :heavy_check_mark: | +| `genesis_verification_key` | - | - | `GENESIS_VERIFICATION_KEY` | Genesis verification key | - | - | :heavy_check_mark: | +| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | +| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | +| `protocol_parameters` | - | - | `PROTOCOL_PARAMETERS__K`, `PROTOCOL_PARAMETERS__M`, and `PROTOCOL_PARAMETERS__PHI_F` | Mithril protocol parameters | - | `{ k: 5, m: 100, phi_f: 0.65 }` | :heavy_check_mark: | +| `run_mode` | `--run-mode` | `-r` | `RUN_MODE` | Runtime mode | `dev` | - | :heavy_check_mark: | +| `store_retention_limit` | - | - | `STORE_RETENTION_LIMIT` | Maximum number of records in stores. If not set, no limit is set. | - | - | - | +| `custom_origin_tag_white_list` | - | - | `CUSTOM_ORIGIN_TAG_WHITE_LIST` | Custom origin tag of client request added to the whitelist (comma separated list). | `EXPLORER,BENCHMARK,CI,NA` | `EXAMPLE` | - | +| `verbose` | `--verbose` | `-v` | `VERBOSE` | Verbosity level | - | Parsed from the number of occurrences: `-v` for `Warning`, `-vv` for `Info`, `-vvv` for `Debug` and `-vvvv` for `Trace` | :heavy_check_mark: | `serve` command: @@ -545,40 +545,40 @@ Here is a list of the available parameters for the serve command: `genesis bootstrap` command: -| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | -| -------------------------- | ------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | -| `genesis_secret_key` | - | - | `GENESIS_SECRET_KEY` | Genesis secret key, :warning: for test only | - | - | :heavy_check_mark: | -| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | -| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | -| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | -| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | -| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | +| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | +| -------------------------- | ------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | +| `genesis_secret_key` | - | - | `GENESIS_SECRET_KEY` | Genesis secret key, :warning: for test only | - | - | :heavy_check_mark: | +| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | +| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | +| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | +| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | +| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | `genesis export` command: -| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | -| -------------------------- | ------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | -| `target_path` | `--target-path` | - | - | Path of the file to export the payload to. | - | - | :heavy_check_mark: | -| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | -| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | -| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | -| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | -| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | +| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | +| -------------------------- | ------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | +| `target_path` | `--target-path` | - | - | Path of the file to export the payload to. | - | - | :heavy_check_mark: | +| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | +| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | +| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | +| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | +| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | `genesis import` command: -| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | -| -------------------------- | ---------------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | -| `signed_payload_path` | `--signed-payload-path` | - | - | Path of the payload to import. | - | - | :heavy_check_mark: | -| `genesis_verification_key` | `--genesis-verification-key` | - | - | Genesis verification key | - | - | :heavy_check_mark: | -| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | -| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | -| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | -| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | -| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | +| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory | +| -------------------------- | ---------------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------ | ------------- | ---------------------------------- | :----------------: | +| `signed_payload_path` | `--signed-payload-path` | - | - | Path of the payload to import. | - | - | :heavy_check_mark: | +| `genesis_verification_key` | `--genesis-verification-key` | - | - | Genesis verification key | - | - | :heavy_check_mark: | +| `data_stores_directory` | - | - | `DATA_STORES_DIRECTORY` | Directory to store aggregator databases | - | `./mithril-aggregator/stores` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | +| `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | - | +| `chain_observer_type` | - | - | `CHAIN_OBSERVER_TYPE` | Chain observer type that can be `cardano-cli`, `pallas` or `fake`. | `pallas` | - | :heavy_check_mark: | +| `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | +| `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | `genesis sign` command: diff --git a/docs/website/versioned_docs/version-maintained/manual/develop/nodes/mithril-signer.md b/docs/website/versioned_docs/version-maintained/manual/develop/nodes/mithril-signer.md index 1d2fcc15922..df6b0a08e15 100644 --- a/docs/website/versioned_docs/version-maintained/manual/develop/nodes/mithril-signer.md +++ b/docs/website/versioned_docs/version-maintained/manual/develop/nodes/mithril-signer.md @@ -236,7 +236,7 @@ Here is a list of the available parameters: | `verbose` | `--verbose` | `-v` | `VERBOSE` | Verbosity level | - | Parsed from the number of occurrences: `-v` for `Warning`, `-vv` for `Info`, `-vvv` for `Debug` and `-vvvv` for `Trace` | :heavy_check_mark: | | `run_mode` | `--run-mode` | `-r` | `RUN_MODE` | Runtime mode | `dev` | - | :heavy_check_mark: | | `cardano_cli_path` | - | - | `CARDANO_CLI_PATH` | Cardano CLI tool path | - | `cardano-cli` | :heavy_check_mark: | -| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket used by the Cardano CLI tool to communicate with the Cardano node | - | `/tmp/cardano.sock` | :heavy_check_mark: | +| `cardano_node_socket_path` | - | - | `CARDANO_NODE_SOCKET_PATH` | Path of the socket opened by the Cardano node | - | `/ipc/node.socket` | :heavy_check_mark: | | `db_directory` | `--db-directory` | - | `DB_DIRECTORY` | Directory to snapshot from the **Cardano node** | `/db` | - | :heavy_check_mark: | | `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: | | `network_magic` | - | - | `NETWORK_MAGIC` | Cardano network magic number (for `testnet` and `devnet`) | - | `1097911063` or `42` | - | diff --git a/mithril-aggregator/config/dev.json b/mithril-aggregator/config/dev.json index e9beb33babf..d950ad05650 100644 --- a/mithril-aggregator/config/dev.json +++ b/mithril-aggregator/config/dev.json @@ -1,7 +1,7 @@ { "environment": "Production", "cardano_cli_path": "cardano-cli", - "cardano_node_socket_path": "/tmp/cardano.sock", + "cardano_node_socket_path": "/ipc/node.socket", "network": "devnet", "network_magic": 42, "run_interval": 30000, diff --git a/mithril-aggregator/config/preview.json b/mithril-aggregator/config/preview.json index 99038a2cfee..b7af38ed360 100644 --- a/mithril-aggregator/config/preview.json +++ b/mithril-aggregator/config/preview.json @@ -1,7 +1,7 @@ { "environment": "Production", "cardano_cli_path": "cardano-cli", - "cardano_node_socket_path": "/tmp/cardano.sock", + "cardano_node_socket_path": "/ipc/node.socket", "network": "preview", "run_interval": 60000, "protocol_parameters": { diff --git a/mithril-aggregator/src/commands/genesis_command.rs b/mithril-aggregator/src/commands/genesis_command.rs index f864569e320..94f6e93578c 100644 --- a/mithril-aggregator/src/commands/genesis_command.rs +++ b/mithril-aggregator/src/commands/genesis_command.rs @@ -27,9 +27,8 @@ pub struct GenesisCommandConfiguration { #[example = "`cardano-cli`"] pub cardano_cli_path: Option, - /// Path of the socket used by the Cardano CLI tool - /// to communicate with the Cardano node - #[example = "`/tmp/cardano.sock`"] + /// Path of the socket opened by the Cardano node + #[example = "`/ipc/node.socket`"] pub cardano_node_socket_path: PathBuf, /// Cardano Network Magic number From d44f5dc54bd8d284a9eeabd8ad696dc089627b24 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 27 Jun 2025 14:49:45 +0200 Subject: [PATCH 25/38] chore: apply review comments --- internal/mithril-dmq-node/src/consumer/pallas.rs | 10 +++++++--- internal/mithril-dmq-node/src/publisher/pallas.rs | 8 ++++---- mithril-common/src/messages/register_signature.rs | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/internal/mithril-dmq-node/src/consumer/pallas.rs b/internal/mithril-dmq-node/src/consumer/pallas.rs index 15ae91a7c1d..62984a5b6d3 100644 --- a/internal/mithril-dmq-node/src/consumer/pallas.rs +++ b/internal/mithril-dmq-node/src/consumer/pallas.rs @@ -48,7 +48,7 @@ impl DmqConsumerPallas { DmqClient::connect(&self.socket, self.network.code()) .await .map_err(|err| anyhow!(err)) - .with_context(|| "PallasChainReader failed to create a new client") + .with_context(|| "DmqConsumerPallas failed to create a new client") } /// Gets the cached `DmqClient`, creating a new one if it does not exist. @@ -71,7 +71,7 @@ impl DmqConsumerPallas { async fn drop_client(&self) -> StdResult<()> { debug!( self.logger, - "Drop exsiting DMQ client"; + "Drop existing DMQ client"; "socket" => ?self.socket, "network" => ?self.network ); @@ -103,7 +103,11 @@ impl DmqConsumerPallas { .await .map_err(|err| anyhow!("Failed to request notifications from DMQ server: {}", err))?; - let reply = client.msg_notification().recv_next_reply().await.unwrap(); + let reply = client + .msg_notification() + .recv_next_reply() + .await + .map_err(|err| anyhow!("Failed to receive notifications from DMQ server: {}", err))?; debug!(self.logger, "Received single signatures from DMQ"; "messages" => ?reply); if let Err(e) = client.msg_notification().send_done().await { error!(self.logger, "Failed to send Done"; "error" => ?e); diff --git a/internal/mithril-dmq-node/src/publisher/pallas.rs b/internal/mithril-dmq-node/src/publisher/pallas.rs index 372fa6f99b1..e3d174dd7da 100644 --- a/internal/mithril-dmq-node/src/publisher/pallas.rs +++ b/internal/mithril-dmq-node/src/publisher/pallas.rs @@ -44,7 +44,7 @@ impl DmqPublisherPallas { DmqClient::connect(&self.socket, magic) .await .map_err(|err| anyhow!(err)) - .with_context(|| "PallasChainReader failed to create a new client") + .with_context(|| "DmqPublisherPallas failed to create a new client") } } @@ -119,9 +119,9 @@ mod tests { // server waits for request from client and replies to it let request = server_msg.recv_next_request().await.unwrap(); - if let localtxsubmission::Request::Submit(_) = &request { - } else { - panic!("Expected a submit request, but got: {:?}", request); + match &request { + localtxsubmission::Request::Submit(_) => (), + request => panic!("Expected a Submit request, but received: {request:?}"), } let response = if reply_success { localtxsubmission::Response::Accepted diff --git a/mithril-common/src/messages/register_signature.rs b/mithril-common/src/messages/register_signature.rs index 724233bfce3..c7eafb57a4d 100644 --- a/mithril-common/src/messages/register_signature.rs +++ b/mithril-common/src/messages/register_signature.rs @@ -93,7 +93,7 @@ impl RegisterSignatureMessageDmq { } } - /// Convert an `RegisterSignatureMessageDmq` into bytes + /// Convert a `RegisterSignatureMessageDmq` into bytes /// /// # Layout /// * Signed entity type length (u16) From 1e1789fbec6f9b9fd24e53c9343d5d2d8a2ef604 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 16:15:00 +0200 Subject: [PATCH 26/38] refactor(aggregator): gate DMQ consumer behind 'future_dmq' feature --- mithril-aggregator/Cargo.toml | 3 ++- .../builder/enablers/misc.rs | 20 ++++++++++++------- .../src/services/signature_consumer/mod.rs | 2 ++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index d0b2f87f550..0c71ae0f7ad 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -14,6 +14,7 @@ default = ["jemallocator"] bundle_tls = ["reqwest/native-tls-vendored"] jemallocator = ["dep:tikv-jemallocator"] +future_dmq = ["dep:mithril-dmq-node"] [dependencies] anyhow = { workspace = true } @@ -29,7 +30,7 @@ mithril-cardano-node-chain = { path = "../internal/cardano-node/mithril-cardano- mithril-cardano-node-internal-database = { path = "../internal/cardano-node/mithril-cardano-node-internal-database" } mithril-cli-helper = { path = "../internal/mithril-cli-helper" } mithril-common = { path = "../mithril-common", features = ["full"] } -mithril-dmq-node = { path = "../internal/mithril-dmq-node" } +mithril-dmq-node = { path = "../internal/mithril-dmq-node", optional = true } mithril-doc = { path = "../internal/mithril-doc" } mithril-era = { path = "../internal/mithril-era" } mithril-metric = { path = "../internal/mithril-metric" } diff --git a/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs b/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs index a3da44b9144..083c5411cf8 100644 --- a/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs +++ b/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs @@ -7,17 +7,20 @@ use std::sync::Arc; use std::time::Duration; +#[cfg(feature = "future_dmq")] use mithril_common::messages::RegisterSignatureMessageDmq; +#[cfg(feature = "future_dmq")] use mithril_dmq_node::DmqConsumerPallas; use mithril_signed_entity_lock::SignedEntityTypeLock; use crate::database::repository::CertificateRepository; use crate::dependency_injection::{DependenciesBuilder, Result}; use crate::get_dependency; +#[cfg(feature = "future_dmq")] +use crate::services::SignatureConsumerDmq; use crate::services::{ AggregatorClient, AggregatorHTTPClient, MessageService, MithrilMessageService, - SequentialSignatureProcessor, SignatureConsumer, SignatureConsumerDmq, SignatureConsumerNoop, - SignatureProcessor, + SequentialSignatureProcessor, SignatureConsumer, SignatureConsumerNoop, SignatureProcessor, }; impl DependenciesBuilder { async fn build_signed_entity_type_lock(&mut self) -> Result> { @@ -77,17 +80,20 @@ impl DependenciesBuilder { /// Builds a [SignatureConsumer] pub async fn build_signature_consumer(&mut self) -> Result> { - let signature_consumer = - if let Some(dmq_node_socket_path) = self.configuration.dmq_node_socket_path() { + #[cfg(feature = "future_dmq")] + let signature_consumer = match self.configuration.dmq_node_socket_path() { + Some(dmq_node_socket_path) => { let dmq_consumer = Arc::new(DmqConsumerPallas::::new( dmq_node_socket_path, self.configuration.get_network()?, self.root_logger(), )); Arc::new(SignatureConsumerDmq::new(dmq_consumer)) as Arc - } else { - Arc::new(SignatureConsumerNoop) as Arc - }; + } + _ => Arc::new(SignatureConsumerNoop) as Arc, + }; + #[cfg(not(feature = "future_dmq"))] + let signature_consumer = Arc::new(SignatureConsumerNoop) as Arc; Ok(signature_consumer) } diff --git a/mithril-aggregator/src/services/signature_consumer/mod.rs b/mithril-aggregator/src/services/signature_consumer/mod.rs index 3b5068680a0..52522ee135e 100644 --- a/mithril-aggregator/src/services/signature_consumer/mod.rs +++ b/mithril-aggregator/src/services/signature_consumer/mod.rs @@ -1,8 +1,10 @@ +#[cfg(feature = "future_dmq")] mod dmq; mod fake; mod interface; mod noop; +#[cfg(feature = "future_dmq")] pub use dmq::*; pub use fake::*; pub use interface::*; From 93593d0b28ec0720daf12affc41aa6ecd055fad9 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 16:19:18 +0200 Subject: [PATCH 27/38] refactor(signer): gate DMQ publisher behind 'future_dmq' feature --- mithril-signer/Cargo.toml | 3 +- .../src/dependency_injection/builder.rs | 57 ++++++++++++------- .../src/services/signature_publisher/mod.rs | 2 + 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/mithril-signer/Cargo.toml b/mithril-signer/Cargo.toml index 5aad717e079..919533968f2 100644 --- a/mithril-signer/Cargo.toml +++ b/mithril-signer/Cargo.toml @@ -14,6 +14,7 @@ default = ["jemallocator"] bundle_tls = ["reqwest/native-tls-vendored"] jemallocator = ["dep:tikv-jemallocator"] +future_dmq = ["dep:mithril-dmq-node"] [dependencies] anyhow = { workspace = true } @@ -26,7 +27,7 @@ mithril-cardano-node-chain = { path = "../internal/cardano-node/mithril-cardano- mithril-cardano-node-internal-database = { path = "../internal/cardano-node/mithril-cardano-node-internal-database" } mithril-cli-helper = { path = "../internal/mithril-cli-helper" } mithril-common = { path = "../mithril-common", features = ["full"] } -mithril-dmq-node = { path = "../internal/mithril-dmq-node" } +mithril-dmq-node = { path = "../internal/mithril-dmq-node", optional = true } mithril-doc = { path = "../internal/mithril-doc" } mithril-era = { path = "../internal/mithril-era" } mithril-metric = { path = "../internal/mithril-metric" } diff --git a/mithril-signer/src/dependency_injection/builder.rs b/mithril-signer/src/dependency_injection/builder.rs index d71d5fb0ef9..4e98c6a5e0d 100644 --- a/mithril-signer/src/dependency_injection/builder.rs +++ b/mithril-signer/src/dependency_injection/builder.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, Context}; -use mithril_dmq_node::{DmqMessageBuilder, DmqPublisherPallas}; use slog::Logger; use tokio::sync::{Mutex, RwLock}; @@ -20,16 +19,18 @@ use mithril_cardano_node_internal_database::{ signable_builder::{CardanoDatabaseSignableBuilder, CardanoImmutableFilesFullSignableBuilder}, ImmutableFileObserver, ImmutableFileSystemObserver, }; +use mithril_common::api_version::APIVersionProvider; use mithril_common::crypto_helper::{ KesSigner, KesSignerStandard, OpCert, ProtocolPartyId, SerDeShelleyFileFormat, }; +#[cfg(feature = "future_dmq")] +use mithril_common::messages::RegisterSignatureMessageDmq; use mithril_common::signable_builder::{ CardanoStakeDistributionSignableBuilder, CardanoTransactionsSignableBuilder, MithrilSignableBuilderService, MithrilStakeDistributionSignableBuilder, SignableBuilderServiceDependencies, }; use mithril_common::StdResult; -use mithril_common::{api_version::APIVersionProvider, messages::RegisterSignatureMessageDmq}; use mithril_era::{EraChecker, EraReader}; use mithril_signed_entity_lock::SignedEntityTypeLock; @@ -40,6 +41,12 @@ use mithril_persistence::database::repository::CardanoTransactionRepository; use mithril_persistence::database::{ApplicationNodeType, SqlMigration}; use mithril_persistence::sqlite::{ConnectionBuilder, SqliteConnection, SqliteConnectionPool}; +#[cfg(feature = "future_dmq")] +use mithril_dmq_node::{DmqMessageBuilder, DmqPublisherPallas}; + +use crate::dependency_injection::SignerDependencyContainer; +#[cfg(feature = "future_dmq")] +use crate::services::SignaturePublisherDmq; use crate::services::{ AggregatorHTTPClient, CardanoTransactionsImporter, CardanoTransactionsPreloaderActivationSigner, MithrilEpochService, MithrilSingleSigner, @@ -53,7 +60,6 @@ use crate::{ database::repository::{ProtocolInitializerRepository, SignedBeaconRepository, StakePoolStore}, services::SignaturePublisher, }; -use crate::{dependency_injection::SignerDependencyContainer, services::SignaturePublisherDmq}; use crate::{ Configuration, MetricsService, HTTP_REQUEST_TIMEOUT_DURATION, SQLITE_FILE, SQLITE_FILE_CARDANO_TRANSACTION, @@ -412,27 +418,34 @@ impl<'a> DependenciesBuilder<'a> { _ => None, }; - let cardano_network = &self.config.get_network()?; - let signature_publisher = { let first_publisher = SignaturePublisherRetrier::new( - if let Some(dmq_node_socket_path) = &self.config.dmq_node_socket_path { - let dmq_message_builder = DmqMessageBuilder::new_with_default_ttl( - kes_signer - .clone() - .ok_or(anyhow!("A KES signer is mandatory to sign DMQ messages"))?, - chain_observer.clone(), - ); - Arc::new(SignaturePublisherDmq::new(Arc::new(DmqPublisherPallas::< - RegisterSignatureMessageDmq, - >::new( - dmq_node_socket_path.to_owned(), - *cardano_network, - dmq_message_builder, - self.root_logger(), - )))) as Arc - } else { - Arc::new(SignaturePublisherNoop) as Arc + { + #[cfg(feature = "future_dmq")] + let publisher = match &self.config.dmq_node_socket_path { + Some(dmq_node_socket_path) => { + let cardano_network = &self.config.get_network()?; + let dmq_message_builder = DmqMessageBuilder::new_with_default_ttl( + kes_signer.clone().ok_or(anyhow!( + "A KES signer is mandatory to sign DMQ messages" + ))?, + chain_observer.clone(), + ); + Arc::new(SignaturePublisherDmq::new(Arc::new(DmqPublisherPallas::< + RegisterSignatureMessageDmq, + >::new( + dmq_node_socket_path.to_owned(), + *cardano_network, + dmq_message_builder, + self.root_logger(), + )))) as Arc + } + _ => Arc::new(SignaturePublisherNoop) as Arc, + }; + #[cfg(not(feature = "future_dmq"))] + let publisher = Arc::new(SignaturePublisherNoop) as Arc; + + publisher }, SignaturePublishRetryPolicy::never(), ); diff --git a/mithril-signer/src/services/signature_publisher/mod.rs b/mithril-signer/src/services/signature_publisher/mod.rs index 099749b5b22..def982fda6f 100644 --- a/mithril-signer/src/services/signature_publisher/mod.rs +++ b/mithril-signer/src/services/signature_publisher/mod.rs @@ -1,4 +1,5 @@ mod delayer; +#[cfg(feature = "future_dmq")] mod dmq; mod http; mod interface; @@ -6,6 +7,7 @@ mod noop; mod retrier; pub use delayer::*; +#[cfg(feature = "future_dmq")] pub use dmq::*; pub use interface::*; pub use noop::*; From 61a586a795b04e38860f48c22c22e10c6e1edfc0 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 16:21:40 +0200 Subject: [PATCH 28/38] chore(ci): add 'future_dqm' feature to Ubuntu tests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22f2422a952..c00f5e6c5bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,7 +177,7 @@ jobs: include: - os: ubuntu-24.04 - test-args: --features full,unstable --workspace + test-args: --features full,unstable,future_dmq --workspace # Exclude nodes not officially supported on Windows and macOS (only mithril-client is supported) - os: macos-14 test-args: > From a70a0d9133d94638ac3924e8fe8b3eb2b5c2d135 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 16:24:33 +0200 Subject: [PATCH 29/38] refactor(dmq): better anyhow usage Co-authored-by: DJO --- internal/mithril-dmq-node/src/consumer/pallas.rs | 9 ++++----- internal/mithril-dmq-node/src/message.rs | 2 +- internal/mithril-dmq-node/src/publisher/pallas.rs | 7 +++---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/mithril-dmq-node/src/consumer/pallas.rs b/internal/mithril-dmq-node/src/consumer/pallas.rs index 62984a5b6d3..c6438797a1c 100644 --- a/internal/mithril-dmq-node/src/consumer/pallas.rs +++ b/internal/mithril-dmq-node/src/consumer/pallas.rs @@ -47,7 +47,6 @@ impl DmqConsumerPallas { ); DmqClient::connect(&self.socket, self.network.code()) .await - .map_err(|err| anyhow!(err)) .with_context(|| "DmqConsumerPallas failed to create a new client") } @@ -101,13 +100,13 @@ impl DmqConsumerPallas { .msg_notification() .send_request_messages_blocking() .await - .map_err(|err| anyhow!("Failed to request notifications from DMQ server: {}", err))?; + .with_context(|| "Failed to request notifications from DMQ server: {}")?; let reply = client .msg_notification() .recv_next_reply() .await - .map_err(|err| anyhow!("Failed to receive notifications from DMQ server: {}", err))?; + .with_context(|| "Failed to receive notifications from DMQ server")?; debug!(self.logger, "Received single signatures from DMQ"; "messages" => ?reply); if let Err(e) = client.msg_notification().send_done().await { error!(self.logger, "Failed to send Done"; "error" => ?e); @@ -118,10 +117,10 @@ impl DmqConsumerPallas { .into_iter() .map(|dmq_message| { let opcert = OpCert::try_from_bytes(&dmq_message.operational_certificate) - .map_err(|e| anyhow!("Failed to parse operational certificate: {}", e))?; + .with_context(|| "Failed to parse operational certificate")?; let party_id = opcert.compute_protocol_party_id()?; let payload = M::try_from_bytes(&dmq_message.msg_body) - .map_err(|e| anyhow!("Failed to parse DMQ message body: {}", e))?; + .with_context(|| "Failed to parse DMQ message body")?; Ok((payload, party_id)) }) diff --git a/internal/mithril-dmq-node/src/message.rs b/internal/mithril-dmq-node/src/message.rs index 2d88ab90b7e..934f1ed350e 100644 --- a/internal/mithril-dmq-node/src/message.rs +++ b/internal/mithril-dmq-node/src/message.rs @@ -70,7 +70,7 @@ impl DmqMessageBuilder { .block_number; let block_number = (*block_number) .try_into() - .map_err(|_| anyhow!("Failed to convert block number to u32"))?; + .with_context(|| "Failed to convert block number to u32")?; let (kes_signature, operational_certificate) = self .kes_signer .sign(message_bytes, block_number) diff --git a/internal/mithril-dmq-node/src/publisher/pallas.rs b/internal/mithril-dmq-node/src/publisher/pallas.rs index e3d174dd7da..0a9ab3ca1f5 100644 --- a/internal/mithril-dmq-node/src/publisher/pallas.rs +++ b/internal/mithril-dmq-node/src/publisher/pallas.rs @@ -1,6 +1,6 @@ use std::{fmt::Debug, marker::PhantomData, path::PathBuf}; -use anyhow::{anyhow, Context}; +use anyhow::Context; use pallas_network::{facades::DmqClient, miniprotocols::localtxsubmission::Response}; use slog::{debug, error, Logger}; @@ -43,7 +43,6 @@ impl DmqPublisherPallas { let magic = self.network.code(); DmqClient::connect(&self.socket, magic) .await - .map_err(|err| anyhow!(err)) .with_context(|| "DmqPublisherPallas failed to create a new client") } } @@ -67,14 +66,14 @@ impl DmqPublisher for DmqPublisherPallas .msg_submission() .send_submit_tx(dmq_message) .await - .map_err(|err| anyhow!("Failed to submit DMQ message: {}", err))?; + .with_context(|| "Failed to submit DMQ message")?; let response = client.msg_submission().recv_submit_tx_response().await?; if let Err(e) = client.msg_submission().terminate_gracefully().await { error!(self.logger, "Failed to send Done"; "error" => ?e); } if response != Response::Accepted { - return Err(anyhow!("Failed to publish DMQ message: {:?}", response)); + anyhow::bail!("Failed to publish DMQ message: {:?}", response); } Ok(()) From cd385981803a7c0ef6785ff394935d3f20711bc5 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 18:20:13 +0200 Subject: [PATCH 30/38] refactor(common): remove unnecessary 'map_err' with 'bincode' deserialization --- mithril-common/src/crypto_helper/merkle_map.rs | 3 +-- mithril-common/src/crypto_helper/merkle_tree.rs | 3 +-- mithril-common/src/entities/signed_entity_type.rs | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/mithril-common/src/crypto_helper/merkle_map.rs b/mithril-common/src/crypto_helper/merkle_map.rs index fd8f5b0beba..9e680b869b4 100644 --- a/mithril-common/src/crypto_helper/merkle_map.rs +++ b/mithril-common/src/crypto_helper/merkle_map.rs @@ -360,8 +360,7 @@ impl Deserialize<'de>> MKMapProof { /// Convert the proof from bytes pub fn from_bytes(bytes: &[u8]) -> StdResult { let (res, _) = - bincode::serde::decode_from_slice::(bytes, bincode::config::standard()) - .map_err(|e| anyhow!(e))?; + bincode::serde::decode_from_slice::(bytes, bincode::config::standard())?; Ok(res) } diff --git a/mithril-common/src/crypto_helper/merkle_tree.rs b/mithril-common/src/crypto_helper/merkle_tree.rs index 85c4f87d959..80f447af734 100644 --- a/mithril-common/src/crypto_helper/merkle_tree.rs +++ b/mithril-common/src/crypto_helper/merkle_tree.rs @@ -204,8 +204,7 @@ impl MKProof { /// Convert the proof from bytes pub fn from_bytes(bytes: &[u8]) -> StdResult { let (res, _) = - bincode::serde::decode_from_slice::(bytes, bincode::config::standard()) - .map_err(|e| anyhow!(e))?; + bincode::serde::decode_from_slice::(bytes, bincode::config::standard())?; Ok(res) } diff --git a/mithril-common/src/entities/signed_entity_type.rs b/mithril-common/src/entities/signed_entity_type.rs index 68c56ec3f90..fef43ec9c7b 100644 --- a/mithril-common/src/entities/signed_entity_type.rs +++ b/mithril-common/src/entities/signed_entity_type.rs @@ -160,8 +160,7 @@ impl SignedEntityType { impl TryFromBytes for SignedEntityType { fn try_from_bytes(bytes: &[u8]) -> StdResult { let (res, _) = - bincode::serde::decode_from_slice::(bytes, bincode::config::standard()) - .map_err(|e| anyhow!(e))?; + bincode::serde::decode_from_slice::(bytes, bincode::config::standard())?; Ok(res) } From 60c58c3d5017b20e36eca033313d15b9548d8374 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 18:23:54 +0200 Subject: [PATCH 31/38] chore(common): enhance readability of 'RegisterSignatureMessageDmq' bytes decoder --- mithril-common/src/messages/register_signature.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mithril-common/src/messages/register_signature.rs b/mithril-common/src/messages/register_signature.rs index c7eafb57a4d..5f6dbf2223f 100644 --- a/mithril-common/src/messages/register_signature.rs +++ b/mithril-common/src/messages/register_signature.rs @@ -116,16 +116,18 @@ impl RegisterSignatureMessageDmq { /// Extract a `RegisterSignatureMessageDmq` from bytes. pub fn try_from_bytes_vec(bytes: &[u8]) -> StdResult { + const SIGNED_ENTITY_TYPE_LENGTH_BYTES_SIZE: usize = 2; + const SIGNATURE_LENGTH_BYTES_SIZE: usize = 4; let mut bytes_index = 0; - let mut u16_bytes = [0u8; 2]; + let mut u16_bytes = [0u8; SIGNED_ENTITY_TYPE_LENGTH_BYTES_SIZE]; u16_bytes.copy_from_slice( bytes - .get(bytes_index..bytes_index + 2) + .get(bytes_index..bytes_index + SIGNED_ENTITY_TYPE_LENGTH_BYTES_SIZE) .ok_or(anyhow!("Failed to read `Signed entity type length` bytes"))?, ); let signed_entity_bytes_length = u16::from_be_bytes(u16_bytes) as usize; - bytes_index += 2; + bytes_index += SIGNED_ENTITY_TYPE_LENGTH_BYTES_SIZE; let signed_entity_bytes = bytes .get(bytes_index..bytes_index + signed_entity_bytes_length) @@ -133,14 +135,14 @@ impl RegisterSignatureMessageDmq { let signed_entity_type = SignedEntityType::try_from_bytes(signed_entity_bytes)?; bytes_index += signed_entity_bytes_length; - let mut u32_bytes = [0u8; 4]; + let mut u32_bytes = [0u8; SIGNATURE_LENGTH_BYTES_SIZE]; u32_bytes.copy_from_slice( bytes - .get(bytes_index..bytes_index + 4) + .get(bytes_index..bytes_index + SIGNATURE_LENGTH_BYTES_SIZE) .ok_or(anyhow!("Failed to read `Signature length` bytes"))?, ); let signature_bytes_length = u32::from_be_bytes(u32_bytes) as usize; - bytes_index += 4; + bytes_index += SIGNATURE_LENGTH_BYTES_SIZE; let signature_bytes = bytes .get(bytes_index..bytes_index + signature_bytes_length) From 1d251edb5e9fec9c7181fdd76907237b1a46f667 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 18:26:56 +0200 Subject: [PATCH 32/38] refactor(dmq): better TTL support in 'DmqMessageBuilder' --- internal/mithril-dmq-node/src/message.rs | 25 ++++++------------- .../mithril-dmq-node/src/publisher/pallas.rs | 8 +++--- .../src/dependency_injection/builder.rs | 2 +- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/internal/mithril-dmq-node/src/message.rs b/internal/mithril-dmq-node/src/message.rs index 934f1ed350e..8b05dab8263 100644 --- a/internal/mithril-dmq-node/src/message.rs +++ b/internal/mithril-dmq-node/src/message.rs @@ -22,28 +22,19 @@ pub struct DmqMessageBuilder { impl DmqMessageBuilder { /// Creates a new instance of `DmqMessageBuilder`. - pub fn new( - kes_signer: Arc, - chain_observer: Arc, - ttl_blocks: u16, - ) -> Self { + pub fn new(kes_signer: Arc, chain_observer: Arc) -> Self { Self { kes_signer, chain_observer, - ttl_blocks, + ttl_blocks: DMQ_MESSAGE_TTL_IN_BLOCKS, } } - /// Creates a new instance of `DmqMessageBuilder` with default TTL. - pub fn new_with_default_ttl( - kes_signer: Arc, - chain_observer: Arc, - ) -> Self { - Self { - kes_signer, - chain_observer, - ttl_blocks: DMQ_MESSAGE_TTL_IN_BLOCKS, - } + /// Set the TTL (Time To Live) for DMQ messages in blocks. + pub fn set_ttl(mut self, ttl_blocks: u16) -> Self { + self.ttl_blocks = ttl_blocks; + + self } /// Builds a DMQ message from the provided message bytes. @@ -127,7 +118,7 @@ mod tests { }, ..TimePoint::dummy() }))); - let builder = DmqMessageBuilder::new(kes_signer, chain_observer, 100); + let builder = DmqMessageBuilder::new(kes_signer, chain_observer).set_ttl(100); let message = test_utils::TestMessage { content: b"test".to_vec(), }; diff --git a/internal/mithril-dmq-node/src/publisher/pallas.rs b/internal/mithril-dmq-node/src/publisher/pallas.rs index 0a9ab3ca1f5..3d9a57dc19a 100644 --- a/internal/mithril-dmq-node/src/publisher/pallas.rs +++ b/internal/mithril-dmq-node/src/publisher/pallas.rs @@ -159,8 +159,8 @@ mod tests { Arc::new(kes_signer) }, Arc::new(FakeChainObserver::default()), - 100, - ), + ) + .set_ttl(100), TestLogger::stdout(), ); @@ -195,8 +195,8 @@ mod tests { Arc::new(kes_signer) }, Arc::new(FakeChainObserver::default()), - 100, - ), + ) + .set_ttl(100), TestLogger::stdout(), ); diff --git a/mithril-signer/src/dependency_injection/builder.rs b/mithril-signer/src/dependency_injection/builder.rs index 4e98c6a5e0d..59ab3ae0903 100644 --- a/mithril-signer/src/dependency_injection/builder.rs +++ b/mithril-signer/src/dependency_injection/builder.rs @@ -425,7 +425,7 @@ impl<'a> DependenciesBuilder<'a> { let publisher = match &self.config.dmq_node_socket_path { Some(dmq_node_socket_path) => { let cardano_network = &self.config.get_network()?; - let dmq_message_builder = DmqMessageBuilder::new_with_default_ttl( + let dmq_message_builder = DmqMessageBuilder::new( kes_signer.clone().ok_or(anyhow!( "A KES signer is mandatory to sign DMQ messages" ))?, From 11a9726cbacad8caca3fb2b35dadac59ade5e01e Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 18:38:10 +0200 Subject: [PATCH 33/38] refactor(common): 'tests_setup' behind 'test_tools' feature --- mithril-common/src/crypto_helper/cardano/kes/mod.rs | 11 +++++++---- .../src/crypto_helper/cardano/kes/signer_fake.rs | 9 ++++++--- .../src/crypto_helper/cardano/kes/signer_with_key.rs | 9 ++++++--- .../crypto_helper/cardano/kes/verifier_standard.rs | 9 ++++++--- .../src/crypto_helper/cardano/key_certification.rs | 12 ++++++++---- mithril-common/src/crypto_helper/mod.rs | 7 +++++-- 6 files changed, 38 insertions(+), 19 deletions(-) diff --git a/mithril-common/src/crypto_helper/cardano/kes/mod.rs b/mithril-common/src/crypto_helper/cardano/kes/mod.rs index 9365d31d57c..b555fe2843f 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/mod.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/mod.rs @@ -1,13 +1,16 @@ mod error; mod interface; -mod signer_fake; mod signer_with_key; -pub mod tests_setup; mod verifier_standard; pub use error::*; pub use interface::*; -pub use signer_fake::*; pub use signer_with_key::*; -pub use tests_setup::*; pub use verifier_standard::*; + +cfg_test_tools! { + mod signer_fake; + pub mod tests_setup; + + pub use signer_fake::*; +} diff --git a/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs b/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs index ba0613b327c..935eeefbc75 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/signer_fake.rs @@ -5,9 +5,12 @@ use std::sync::Mutex; use crate::{ crypto_helper::{ - cardano::{ - create_kes_cryptographic_material, KesCryptographicMaterialForTest, - KesPartyIndexForTest, KesSignerStandard, + cardano::kes::{ + tests_setup::{ + create_kes_cryptographic_material, KesCryptographicMaterialForTest, + KesPartyIndexForTest, + }, + KesSignerStandard, }, KesPeriod, KesSigner, OpCert, }, diff --git a/mithril-common/src/crypto_helper/cardano/kes/signer_with_key.rs b/mithril-common/src/crypto_helper/cardano/kes/signer_with_key.rs index 45b59505826..1d1e08dd4e4 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/signer_with_key.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/signer_with_key.rs @@ -65,9 +65,12 @@ impl KesSigner for KesSignerStandard { mod tests { use super::*; - use crate::crypto_helper::cardano::{ - kes::tests_setup::{create_kes_cryptographic_material, KesCryptographicMaterialForTest}, - KesPartyIndexForTest, KesVerifier, KesVerifierStandard, + use crate::crypto_helper::cardano::kes::{ + tests_setup::{ + create_kes_cryptographic_material, KesCryptographicMaterialForTest, + KesPartyIndexForTest, + }, + KesVerifier, KesVerifierStandard, }; #[test] diff --git a/mithril-common/src/crypto_helper/cardano/kes/verifier_standard.rs b/mithril-common/src/crypto_helper/cardano/kes/verifier_standard.rs index f9d05d196a2..82cc0a222ed 100644 --- a/mithril-common/src/crypto_helper/cardano/kes/verifier_standard.rs +++ b/mithril-common/src/crypto_helper/cardano/kes/verifier_standard.rs @@ -47,9 +47,12 @@ impl KesVerifier for KesVerifierStandard { #[cfg(test)] mod tests { - use crate::crypto_helper::cardano::{ - kes::tests_setup::{create_kes_cryptographic_material, KesCryptographicMaterialForTest}, - KesPartyIndexForTest, KesSigner, KesSignerStandard, + use crate::crypto_helper::cardano::kes::{ + tests_setup::{ + create_kes_cryptographic_material, KesCryptographicMaterialForTest, + KesPartyIndexForTest, + }, + KesSigner, KesSignerStandard, }; use super::*; diff --git a/mithril-common/src/crypto_helper/cardano/key_certification.rs b/mithril-common/src/crypto_helper/cardano/key_certification.rs index e173ac5147d..fdacbe0e447 100644 --- a/mithril-common/src/crypto_helper/cardano/key_certification.rs +++ b/mithril-common/src/crypto_helper/cardano/key_certification.rs @@ -299,16 +299,20 @@ impl KeyRegWrapper { #[cfg(test)] mod test { - use super::*; - use crate::crypto_helper::cardano::{ - kes::tests_setup::{create_kes_cryptographic_material, KesCryptographicMaterialForTest}, - KesPartyIndexForTest, KesSignerStandard, + use crate::crypto_helper::cardano::kes::{ + tests_setup::{ + create_kes_cryptographic_material, KesCryptographicMaterialForTest, + KesPartyIndexForTest, + }, + KesSignerStandard, }; use crate::crypto_helper::{OpCert, SerDeShelleyFileFormat}; use rand_chacha::ChaCha20Rng; use rand_core::SeedableRng; + use super::*; + #[test] fn test_vector_key_reg() { let params = StmParameters { diff --git a/mithril-common/src/crypto_helper/mod.rs b/mithril-common/src/crypto_helper/mod.rs index 9d1c8057098..6ae2dc7615b 100644 --- a/mithril-common/src/crypto_helper/mod.rs +++ b/mithril-common/src/crypto_helper/mod.rs @@ -16,10 +16,13 @@ cfg_test_tools! { pub use cardano::ColdKeyGenerator; pub use cardano::{ - KesPeriod, KesSigner, KesSignerFake, KesSignerStandard, KesVerifier, KesVerifierStandard, - KesVerifyError, OpCert, ProtocolInitializerErrorWrapper, ProtocolRegistrationErrorWrapper, + KesPeriod, KesSigner, KesSignerStandard, KesVerifier, KesVerifierStandard, KesVerifyError, + OpCert, ProtocolInitializerErrorWrapper, ProtocolRegistrationErrorWrapper, SerDeShelleyFileFormat, Sum6KesBytes, }; +cfg_test_tools! { + pub use cardano::KesSignerFake; +} pub use codec::*; pub use ed25519_alias::{era::*, genesis::*, manifest::*}; pub use merkle_map::{MKMap, MKMapKey, MKMapNode, MKMapProof, MKMapValue}; From 0fd9ca9b2fc451f528401bf235ea10e146e08b17 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 18:41:44 +0200 Subject: [PATCH 34/38] fix(dmq): remove unneeded 'Debug' in 'DmqPublisher' trait --- internal/mithril-dmq-node/src/publisher/interface.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/mithril-dmq-node/src/publisher/interface.rs b/internal/mithril-dmq-node/src/publisher/interface.rs index 4796cc0204a..6f41a23bf3a 100644 --- a/internal/mithril-dmq-node/src/publisher/interface.rs +++ b/internal/mithril-dmq-node/src/publisher/interface.rs @@ -1,11 +1,9 @@ -use std::fmt::Debug; - use mithril_common::{crypto_helper::TryToBytes, StdResult}; /// Trait for publishing messages from a DMQ node. #[cfg_attr(test, mockall::automock)] #[async_trait::async_trait] -pub trait DmqPublisher: Send + Sync { +pub trait DmqPublisher: Send + Sync { /// Publishes a message to the DMQ node. async fn publish_message(&self, message: M) -> StdResult<()>; } From 5b8874b18f6a476f1d17403b42712edeff04c956 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 18:50:09 +0200 Subject: [PATCH 35/38] chore(dmq): rename 'mithril-dmq-node' crate to 'mithril-dmq' --- .github/workflows/ci.yml | 2 +- Cargo.lock | 6 +++--- Cargo.toml | 2 +- Makefile | 2 +- README.md | 5 ++++- internal/{mithril-dmq-node => mithril-dmq}/Cargo.toml | 2 +- internal/{mithril-dmq-node => mithril-dmq}/Makefile | 0 internal/{mithril-dmq-node => mithril-dmq}/README.md | 0 .../src/consumer/interface.rs | 0 .../{mithril-dmq-node => mithril-dmq}/src/consumer/mod.rs | 0 .../src/consumer/pallas.rs | 0 internal/{mithril-dmq-node => mithril-dmq}/src/lib.rs | 0 internal/{mithril-dmq-node => mithril-dmq}/src/message.rs | 0 .../src/publisher/interface.rs | 0 .../{mithril-dmq-node => mithril-dmq}/src/publisher/mod.rs | 0 .../src/publisher/pallas.rs | 0 .../src/test/double/consumer.rs | 0 .../src/test/double/mod.rs | 0 .../src/test/double/publisher.rs | 0 internal/{mithril-dmq-node => mithril-dmq}/src/test/mod.rs | 0 .../{mithril-dmq-node => mithril-dmq}/src/test/payload.rs | 0 mithril-aggregator/Cargo.toml | 4 ++-- .../src/dependency_injection/builder/enablers/misc.rs | 2 +- mithril-aggregator/src/services/signature_consumer/dmq.rs | 4 ++-- mithril-signer/Cargo.toml | 4 ++-- mithril-signer/src/dependency_injection/builder.rs | 2 +- mithril-signer/src/services/signature_publisher/dmq.rs | 4 ++-- 27 files changed, 21 insertions(+), 18 deletions(-) rename internal/{mithril-dmq-node => mithril-dmq}/Cargo.toml (97%) rename internal/{mithril-dmq-node => mithril-dmq}/Makefile (100%) rename internal/{mithril-dmq-node => mithril-dmq}/README.md (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/consumer/interface.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/consumer/mod.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/consumer/pallas.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/lib.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/message.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/publisher/interface.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/publisher/mod.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/publisher/pallas.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/test/double/consumer.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/test/double/mod.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/test/double/publisher.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/test/mod.rs (100%) rename internal/{mithril-dmq-node => mithril-dmq}/src/test/payload.rs (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c00f5e6c5bf..7b2c17e93df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -805,7 +805,7 @@ jobs: # the same name (we only want to document those anyway) cargo doc --no-deps --lib -p mithril-stm -p mithril-common \ -p mithril-cardano-node-chain -p mithril-cardano-node-internal-database \ - -p mithril-dmq-node \ + -p mithril-dmq \ -p mithril-build-script -p mithril-cli-helper -p mithril-doc -p mithril-doc-derive \ -p mithril-era -p mithril-metric -p mithril-persistence -p mithril-resource-pool \ -p mithril-ticker -p mithril-signed-entity-lock -p mithril-signed-entity-preloader \ diff --git a/Cargo.lock b/Cargo.lock index 63a2d2b5b16..1eef3309d9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3929,7 +3929,7 @@ dependencies = [ "mithril-cardano-node-internal-database", "mithril-cli-helper", "mithril-common", - "mithril-dmq-node", + "mithril-dmq", "mithril-doc", "mithril-era", "mithril-metric", @@ -4201,7 +4201,7 @@ dependencies = [ ] [[package]] -name = "mithril-dmq-node" +name = "mithril-dmq" version = "0.1.0" dependencies = [ "anyhow", @@ -4385,7 +4385,7 @@ dependencies = [ "mithril-cardano-node-internal-database", "mithril-cli-helper", "mithril-common", - "mithril-dmq-node", + "mithril-dmq", "mithril-doc", "mithril-era", "mithril-metric", diff --git a/Cargo.toml b/Cargo.toml index ebc25ecb852..018a45120c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ "internal/cardano-node/mithril-cardano-node-internal-database", "internal/mithril-build-script", "internal/mithril-cli-helper", - "internal/mithril-dmq-node", + "internal/mithril-dmq", "internal/mithril-doc", "internal/mithril-doc-derive", "internal/mithril-era", diff --git a/Makefile b/Makefile index c6451c4fd80..3057609c4fc 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ COMPONENTS = mithril-aggregator mithril-client mithril-client-cli mithril-client-wasm \ mithril-common mithril-relay mithril-signer mithril-stm \ internal/mithril-build-script internal/mithril-cli-helper internal/mithril-doc \ - internal/mithril-dmq-node \ + internal/mithril-dmq \ internal/mithril-doc-derive internal/mithril-era internal/mithril-metric internal/mithril-persistence \ internal/mithril-resource-pool internal/mithril-ticker \ internal/cardano-node/mithril-cardano-node-chain internal/cardano-node/mithril-cardano-node-internal-database \ diff --git a/README.md b/README.md index 2886c6369e5..6d7f6ca561f 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ This repository consists of the following parts: - [**Mithril signer**](./mithril-signer): the node of the **Mithril network** responsible for producing individual signatures that are collected and aggregated by the **Mithril aggregator**. - [**Internal**](./internal): the shared tools and API used by **Mithril** crates. + - [**Mithril build script**](./internal/mithril-build-script): a toolbox for Mithril crates that uses a build script phase. - [**Mithril cardano-node-chain**](./internal/cardano-node/mithril-cardano-node-chain): mechanisms to read and interact with the **Cardano chain** through a Cardano node, used by **Mithril network** nodes. @@ -88,7 +89,7 @@ This repository consists of the following parts: - [**Mithril cli helper**](./internal/mithril-cli-helper): **CLI** tools for **Mithril** binaries. - - [**Mithril DMQ node**](./internal/mithril-dmq-node): mechanisms to publish and consume messages of a **Decentralized Message Queue network** through a DMQ node, used by Mithril network nodes. + - [**Mithril DMQ node**](./internal/mithril-dmq): mechanisms to publish and consume messages of a **Decentralized Message Queue network** through a DMQ node, used by Mithril network nodes. - [**Mithril doc**](./internal/mithril-doc): an API that generates Markdown documentation for crate command line arguments. @@ -109,11 +110,13 @@ This repository consists of the following parts: - [**Mithril signed entity prealoader**](./internal/signed-entity/mithril-signed-entity-preloader): a **preload** mechanism for the Cardano transaction signed entity, used by **Mithril network** nodes. - [**tests**](./internal/tests): shared testing tools used by **Mithril** crates. + - [**Mithril api spec**](./internal/tests/mithril-api-spec): toolset to verify conformity of http routes against an Open Api specification, used by **Mithril network** nodes. - [**Mithril test http server**](internal/tests/mithril-test-http-server): provides a test http server, used by **Mithril network** nodes. - [**Mithril test lab**](./mithril-test-lab): the suite of tools that allow us to test and stress the **Mithril** protocol implementations. + - [**Mithril devnet**](./mithril-test-lab/mithril-devnet): the private **Mithril/Cardano network** used to scaffold a **Mithril network** on top of a **Cardano network**. - [**Mithril end to end**](./mithril-test-lab/mithril-end-to-end): the tool used to run test scenarios against a **Mithril devnet**. diff --git a/internal/mithril-dmq-node/Cargo.toml b/internal/mithril-dmq/Cargo.toml similarity index 97% rename from internal/mithril-dmq-node/Cargo.toml rename to internal/mithril-dmq/Cargo.toml index ef61d259ec9..05177fae66b 100644 --- a/internal/mithril-dmq-node/Cargo.toml +++ b/internal/mithril-dmq/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "mithril-dmq-node" +name = "mithril-dmq" description = "Mechanisms to publish and consume messages of a 'Decentralized Message Queue network' through a DMQ node" version = "0.1.0" authors.workspace = true diff --git a/internal/mithril-dmq-node/Makefile b/internal/mithril-dmq/Makefile similarity index 100% rename from internal/mithril-dmq-node/Makefile rename to internal/mithril-dmq/Makefile diff --git a/internal/mithril-dmq-node/README.md b/internal/mithril-dmq/README.md similarity index 100% rename from internal/mithril-dmq-node/README.md rename to internal/mithril-dmq/README.md diff --git a/internal/mithril-dmq-node/src/consumer/interface.rs b/internal/mithril-dmq/src/consumer/interface.rs similarity index 100% rename from internal/mithril-dmq-node/src/consumer/interface.rs rename to internal/mithril-dmq/src/consumer/interface.rs diff --git a/internal/mithril-dmq-node/src/consumer/mod.rs b/internal/mithril-dmq/src/consumer/mod.rs similarity index 100% rename from internal/mithril-dmq-node/src/consumer/mod.rs rename to internal/mithril-dmq/src/consumer/mod.rs diff --git a/internal/mithril-dmq-node/src/consumer/pallas.rs b/internal/mithril-dmq/src/consumer/pallas.rs similarity index 100% rename from internal/mithril-dmq-node/src/consumer/pallas.rs rename to internal/mithril-dmq/src/consumer/pallas.rs diff --git a/internal/mithril-dmq-node/src/lib.rs b/internal/mithril-dmq/src/lib.rs similarity index 100% rename from internal/mithril-dmq-node/src/lib.rs rename to internal/mithril-dmq/src/lib.rs diff --git a/internal/mithril-dmq-node/src/message.rs b/internal/mithril-dmq/src/message.rs similarity index 100% rename from internal/mithril-dmq-node/src/message.rs rename to internal/mithril-dmq/src/message.rs diff --git a/internal/mithril-dmq-node/src/publisher/interface.rs b/internal/mithril-dmq/src/publisher/interface.rs similarity index 100% rename from internal/mithril-dmq-node/src/publisher/interface.rs rename to internal/mithril-dmq/src/publisher/interface.rs diff --git a/internal/mithril-dmq-node/src/publisher/mod.rs b/internal/mithril-dmq/src/publisher/mod.rs similarity index 100% rename from internal/mithril-dmq-node/src/publisher/mod.rs rename to internal/mithril-dmq/src/publisher/mod.rs diff --git a/internal/mithril-dmq-node/src/publisher/pallas.rs b/internal/mithril-dmq/src/publisher/pallas.rs similarity index 100% rename from internal/mithril-dmq-node/src/publisher/pallas.rs rename to internal/mithril-dmq/src/publisher/pallas.rs diff --git a/internal/mithril-dmq-node/src/test/double/consumer.rs b/internal/mithril-dmq/src/test/double/consumer.rs similarity index 100% rename from internal/mithril-dmq-node/src/test/double/consumer.rs rename to internal/mithril-dmq/src/test/double/consumer.rs diff --git a/internal/mithril-dmq-node/src/test/double/mod.rs b/internal/mithril-dmq/src/test/double/mod.rs similarity index 100% rename from internal/mithril-dmq-node/src/test/double/mod.rs rename to internal/mithril-dmq/src/test/double/mod.rs diff --git a/internal/mithril-dmq-node/src/test/double/publisher.rs b/internal/mithril-dmq/src/test/double/publisher.rs similarity index 100% rename from internal/mithril-dmq-node/src/test/double/publisher.rs rename to internal/mithril-dmq/src/test/double/publisher.rs diff --git a/internal/mithril-dmq-node/src/test/mod.rs b/internal/mithril-dmq/src/test/mod.rs similarity index 100% rename from internal/mithril-dmq-node/src/test/mod.rs rename to internal/mithril-dmq/src/test/mod.rs diff --git a/internal/mithril-dmq-node/src/test/payload.rs b/internal/mithril-dmq/src/test/payload.rs similarity index 100% rename from internal/mithril-dmq-node/src/test/payload.rs rename to internal/mithril-dmq/src/test/payload.rs diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index 0c71ae0f7ad..bfd49fc7933 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -14,7 +14,7 @@ default = ["jemallocator"] bundle_tls = ["reqwest/native-tls-vendored"] jemallocator = ["dep:tikv-jemallocator"] -future_dmq = ["dep:mithril-dmq-node"] +future_dmq = ["dep:mithril-dmq"] [dependencies] anyhow = { workspace = true } @@ -30,7 +30,7 @@ mithril-cardano-node-chain = { path = "../internal/cardano-node/mithril-cardano- mithril-cardano-node-internal-database = { path = "../internal/cardano-node/mithril-cardano-node-internal-database" } mithril-cli-helper = { path = "../internal/mithril-cli-helper" } mithril-common = { path = "../mithril-common", features = ["full"] } -mithril-dmq-node = { path = "../internal/mithril-dmq-node", optional = true } +mithril-dmq = { path = "../internal/mithril-dmq", optional = true } mithril-doc = { path = "../internal/mithril-doc" } mithril-era = { path = "../internal/mithril-era" } mithril-metric = { path = "../internal/mithril-metric" } diff --git a/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs b/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs index 083c5411cf8..0c377a35f84 100644 --- a/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs +++ b/mithril-aggregator/src/dependency_injection/builder/enablers/misc.rs @@ -10,7 +10,7 @@ use std::time::Duration; #[cfg(feature = "future_dmq")] use mithril_common::messages::RegisterSignatureMessageDmq; #[cfg(feature = "future_dmq")] -use mithril_dmq_node::DmqConsumerPallas; +use mithril_dmq::DmqConsumerPallas; use mithril_signed_entity_lock::SignedEntityTypeLock; use crate::database::repository::CertificateRepository; diff --git a/mithril-aggregator/src/services/signature_consumer/dmq.rs b/mithril-aggregator/src/services/signature_consumer/dmq.rs index a5fead2e92e..4b14be060d8 100644 --- a/mithril-aggregator/src/services/signature_consumer/dmq.rs +++ b/mithril-aggregator/src/services/signature_consumer/dmq.rs @@ -9,7 +9,7 @@ use mithril_common::{ StdResult, }; -use mithril_dmq_node::DmqConsumer; +use mithril_dmq::DmqConsumer; use super::SignatureConsumer; @@ -56,7 +56,7 @@ impl SignatureConsumer for SignatureConsumerDmq { #[cfg(test)] mod tests { use mithril_common::{crypto_helper::ProtocolSingleSignature, test_utils::fake_keys}; - use mithril_dmq_node::test::double::DmqConsumerFake; + use mithril_dmq::test::double::DmqConsumerFake; use super::*; diff --git a/mithril-signer/Cargo.toml b/mithril-signer/Cargo.toml index 919533968f2..ec37ec72a7a 100644 --- a/mithril-signer/Cargo.toml +++ b/mithril-signer/Cargo.toml @@ -14,7 +14,7 @@ default = ["jemallocator"] bundle_tls = ["reqwest/native-tls-vendored"] jemallocator = ["dep:tikv-jemallocator"] -future_dmq = ["dep:mithril-dmq-node"] +future_dmq = ["dep:mithril-dmq"] [dependencies] anyhow = { workspace = true } @@ -27,7 +27,7 @@ mithril-cardano-node-chain = { path = "../internal/cardano-node/mithril-cardano- mithril-cardano-node-internal-database = { path = "../internal/cardano-node/mithril-cardano-node-internal-database" } mithril-cli-helper = { path = "../internal/mithril-cli-helper" } mithril-common = { path = "../mithril-common", features = ["full"] } -mithril-dmq-node = { path = "../internal/mithril-dmq-node", optional = true } +mithril-dmq = { path = "../internal/mithril-dmq", optional = true } mithril-doc = { path = "../internal/mithril-doc" } mithril-era = { path = "../internal/mithril-era" } mithril-metric = { path = "../internal/mithril-metric" } diff --git a/mithril-signer/src/dependency_injection/builder.rs b/mithril-signer/src/dependency_injection/builder.rs index 59ab3ae0903..5abb7723b2f 100644 --- a/mithril-signer/src/dependency_injection/builder.rs +++ b/mithril-signer/src/dependency_injection/builder.rs @@ -42,7 +42,7 @@ use mithril_persistence::database::{ApplicationNodeType, SqlMigration}; use mithril_persistence::sqlite::{ConnectionBuilder, SqliteConnection, SqliteConnectionPool}; #[cfg(feature = "future_dmq")] -use mithril_dmq_node::{DmqMessageBuilder, DmqPublisherPallas}; +use mithril_dmq::{DmqMessageBuilder, DmqPublisherPallas}; use crate::dependency_injection::SignerDependencyContainer; #[cfg(feature = "future_dmq")] diff --git a/mithril-signer/src/services/signature_publisher/dmq.rs b/mithril-signer/src/services/signature_publisher/dmq.rs index c27c1ab9376..80c88c946e3 100644 --- a/mithril-signer/src/services/signature_publisher/dmq.rs +++ b/mithril-signer/src/services/signature_publisher/dmq.rs @@ -8,7 +8,7 @@ use mithril_common::{ messages::RegisterSignatureMessageDmq, StdResult, }; -use mithril_dmq_node::DmqPublisher; +use mithril_dmq::DmqPublisher; use super::SignaturePublisher; @@ -47,7 +47,7 @@ impl SignaturePublisher for SignaturePublisherDmq { #[cfg(test)] mod tests { use mithril_common::test_utils::fake_data; - use mithril_dmq_node::test::double::DmqPublisherFake; + use mithril_dmq::test::double::DmqPublisherFake; use super::*; From bbd4888866b93d5d6f59bc94ca2bdcc66d8f429a Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 19:32:28 +0200 Subject: [PATCH 36/38] fix(doc): linting of README --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 6d7f6ca561f..c03f744d740 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,6 @@ This repository consists of the following parts: - [**Mithril signer**](./mithril-signer): the node of the **Mithril network** responsible for producing individual signatures that are collected and aggregated by the **Mithril aggregator**. - [**Internal**](./internal): the shared tools and API used by **Mithril** crates. - - [**Mithril build script**](./internal/mithril-build-script): a toolbox for Mithril crates that uses a build script phase. - [**Mithril cardano-node-chain**](./internal/cardano-node/mithril-cardano-node-chain): mechanisms to read and interact with the **Cardano chain** through a Cardano node, used by **Mithril network** nodes. @@ -110,13 +109,11 @@ This repository consists of the following parts: - [**Mithril signed entity prealoader**](./internal/signed-entity/mithril-signed-entity-preloader): a **preload** mechanism for the Cardano transaction signed entity, used by **Mithril network** nodes. - [**tests**](./internal/tests): shared testing tools used by **Mithril** crates. - - [**Mithril api spec**](./internal/tests/mithril-api-spec): toolset to verify conformity of http routes against an Open Api specification, used by **Mithril network** nodes. - [**Mithril test http server**](internal/tests/mithril-test-http-server): provides a test http server, used by **Mithril network** nodes. - [**Mithril test lab**](./mithril-test-lab): the suite of tools that allow us to test and stress the **Mithril** protocol implementations. - - [**Mithril devnet**](./mithril-test-lab/mithril-devnet): the private **Mithril/Cardano network** used to scaffold a **Mithril network** on top of a **Cardano network**. - [**Mithril end to end**](./mithril-test-lab/mithril-end-to-end): the tool used to run test scenarios against a **Mithril devnet**. From 52077da202ccb57cd3147daa45cb55477e403285 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 30 Jun 2025 18:51:52 +0200 Subject: [PATCH 37/38] docs: update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 516c50366c0..8373e2f867f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ As a minor extension, we have adopted a slightly different versioning convention - Abstracted the implementation of KES signature and verification to allow multiple and reusable implementations. +- **UNSTABLE** : + - Support for DMQ signature publisher in the signer and signature consumer in the aggregator. + - Crates versions: | Crate | Version | From 36f6e6737b796eb6f7a7c8405f90a0b3709c6ff3 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 1 Jul 2025 09:51:11 +0200 Subject: [PATCH 38/38] chore: upgrade crate versions * mithril-dmq from `0.1.0` to `0.1.1` * mithril-aggregator from `0.7.69` to `0.7.70` * mithril-common from `0.6.3` to `0.6.4` * mithril-signer from `0.2.256` to `0.2.257` --- Cargo.lock | 8 ++++---- internal/mithril-dmq/Cargo.toml | 2 +- mithril-aggregator/Cargo.toml | 2 +- mithril-common/Cargo.toml | 2 +- mithril-signer/Cargo.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1eef3309d9a..44d0d601941 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3910,7 +3910,7 @@ dependencies = [ [[package]] name = "mithril-aggregator" -version = "0.7.69" +version = "0.7.70" dependencies = [ "anyhow", "async-trait", @@ -4160,7 +4160,7 @@ dependencies = [ [[package]] name = "mithril-common" -version = "0.6.3" +version = "0.6.4" dependencies = [ "anyhow", "async-trait", @@ -4202,7 +4202,7 @@ dependencies = [ [[package]] name = "mithril-dmq" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "async-trait", @@ -4370,7 +4370,7 @@ dependencies = [ [[package]] name = "mithril-signer" -version = "0.2.256" +version = "0.2.257" dependencies = [ "anyhow", "async-trait", diff --git a/internal/mithril-dmq/Cargo.toml b/internal/mithril-dmq/Cargo.toml index 05177fae66b..493a0ae77d2 100644 --- a/internal/mithril-dmq/Cargo.toml +++ b/internal/mithril-dmq/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "mithril-dmq" description = "Mechanisms to publish and consume messages of a 'Decentralized Message Queue network' through a DMQ node" -version = "0.1.0" +version = "0.1.1" authors.workspace = true documentation.workspace = true edition.workspace = true diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index bfd49fc7933..256228e502b 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-aggregator" -version = "0.7.69" +version = "0.7.70" description = "A Mithril Aggregator server" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-common/Cargo.toml b/mithril-common/Cargo.toml index 509dfeefaab..70ca27fcd5a 100644 --- a/mithril-common/Cargo.toml +++ b/mithril-common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-common" -version = "0.6.3" +version = "0.6.4" description = "Common types, interfaces, and utilities for Mithril nodes." authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-signer/Cargo.toml b/mithril-signer/Cargo.toml index ec37ec72a7a..8dc2c7a1d96 100644 --- a/mithril-signer/Cargo.toml +++ b/mithril-signer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-signer" -version = "0.2.256" +version = "0.2.257" description = "A Mithril Signer" authors = { workspace = true } edition = { workspace = true }