From 4368ce01a56757533747615e0aa2f99dac756c8c Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Thu, 16 Oct 2025 16:18:42 +0200 Subject: [PATCH 1/4] Allow offline compilation with `build.rs` overrides You reference a JSON file with the `ACROPOLIS_OFFLINE_MIRROR` environment variable. Each key is an URL that the `build.rs` scripts would normally download. Each value is its contents. If the variable or keys are not set, it downloads as usual. But people compiling without Internet access, can still do so by providing their own pre-downloaded overrides. --- modules/genesis_bootstrapper/Cargo.toml | 1 + modules/genesis_bootstrapper/build.rs | 24 ++++++++++++++++++++---- modules/parameters_state/Cargo.toml | 1 + modules/parameters_state/build.rs | 21 +++++++++++++++++++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/modules/genesis_bootstrapper/Cargo.toml b/modules/genesis_bootstrapper/Cargo.toml index 17fb440e..67a8f856 100644 --- a/modules/genesis_bootstrapper/Cargo.toml +++ b/modules/genesis_bootstrapper/Cargo.toml @@ -26,6 +26,7 @@ tracing = { workspace = true } [build-dependencies] anyhow = { workspace = true } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +serde_json = { workspace = true } tokio = { workspace = true } [lib] diff --git a/modules/genesis_bootstrapper/build.rs b/modules/genesis_bootstrapper/build.rs index 86bce1c5..9c6fa3f5 100644 --- a/modules/genesis_bootstrapper/build.rs +++ b/modules/genesis_bootstrapper/build.rs @@ -1,18 +1,34 @@ // Build-time script to download generics +use std::collections::HashMap; +use std::env; use std::fs; +use std::fs::File; use std::io::Write; use std::path::Path; use anyhow::{Context, Result}; +use serde_json::from_reader; const OUTPUT_DIR: &str = "downloads"; +async fn fetch_bytes(client: &reqwest::Client, url: &str) -> Result> { + if let Ok(path) = env::var("ACROPOLIS_OFFLINE_MIRROR") { + if let Ok(file) = File::open(&path) { + if let Ok(map) = from_reader::<_, HashMap>(file) { + if let Some(s) = map.get(url) { + return Ok(s.as_bytes().to_vec()); + } + } + } + } + let req = client.get(url).build().with_context(|| format!("Failed to request {url}"))?; + let resp = client.execute(req).await.with_context(|| format!("Failed to fetch {url}"))?; + Ok(resp.bytes().await.context("Failed to read response")?.to_vec()) +} + /// Download a URL to a file in OUTPUT_DIR async fn download(client: &reqwest::Client, url: &str, filename: &str) -> Result<()> { - let request = client.get(url).build().with_context(|| format!("Failed to request {url}"))?; - let response = - client.execute(request).await.with_context(|| format!("Failed to fetch {url}"))?; - let data = response.bytes().await.context("Failed to read response")?; + let data = fetch_bytes(client, url).await?; let output_path = Path::new(OUTPUT_DIR); if !output_path.exists() { diff --git a/modules/parameters_state/Cargo.toml b/modules/parameters_state/Cargo.toml index 3b36030e..abeb9516 100644 --- a/modules/parameters_state/Cargo.toml +++ b/modules/parameters_state/Cargo.toml @@ -29,6 +29,7 @@ tracing = { workspace = true } [build-dependencies] reqwest = { version = "0.11", features = ["blocking"] } +serde_json = { workspace = true } [lib] path = "src/parameters_state.rs" diff --git a/modules/parameters_state/build.rs b/modules/parameters_state/build.rs index b6bb61dd..feac6e77 100644 --- a/modules/parameters_state/build.rs +++ b/modules/parameters_state/build.rs @@ -1,16 +1,33 @@ // Build-time script to download generics use reqwest::blocking::get; +use serde_json::from_reader; +use std::collections::HashMap; +use std::env; use std::fs; +use std::fs::File; use std::io::Write; use std::path::Path; const OUTPUT_DIR: &str = "downloads"; +fn fetch_text(url: &str) -> Result> { + if let Ok(path) = env::var("ACROPOLIS_OFFLINE_MIRROR") { + if !path.is_empty() { + if let Ok(file) = File::open(path) { + let map: HashMap = from_reader(file)?; + if let Some(s) = map.get(url) { + return Ok(s.clone()); + } + } + } + } + Ok(get(url)?.error_for_status()?.text()?) +} + /// Download a URL to a file in OUTPUT_DIR fn download(url_base: &str, epoch: &str, filename: &str, rename: &Vec<(&str, &str)>) { let url = format!("{}/{}-genesis.json", url_base, epoch); - let response = get(url).expect("Failed to fetch {url}"); - let mut data = response.text().expect("Failed to read response"); + let mut data = fetch_text(&url).expect("Failed to fetch {url}"); for (what, with) in rename.iter() { data = data.replace(&format!("\"{what}\""), &format!("\"{with}\"")); From 180e10f9672a48079ebe9c067e5c408cc38c31ef Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Fri, 17 Oct 2025 16:24:14 +0200 Subject: [PATCH 2/4] Let user choose their `--config` location; specify defaults in compile time If you set `ACROPOLIS_OMNIBUS_DEFAULT_CONFIG` during compilation, this value will be used as the default, allowing build systems to override the default default of `omnibus.toml`. --- Cargo.lock | 116 +++++++++++++++++++++++++++++++++ Cargo.toml | 1 + processes/omnibus/Cargo.toml | 1 + processes/omnibus/src/main.rs | 11 +++- processes/replayer/Cargo.toml | 1 + processes/replayer/src/main.rs | 52 +++++++++------ 6 files changed, 161 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8113613e..6887ef91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,6 +389,7 @@ dependencies = [ "caryatid_process", "caryatid_sdk", "chrono", + "clap", "config", "opentelemetry", "opentelemetry-otlp", @@ -428,6 +429,7 @@ dependencies = [ "caryatid_process", "caryatid_sdk", "chrono", + "clap", "config", "serde", "serde_json", @@ -589,6 +591,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", +] + [[package]] name = "anyhow" version = "1.0.100" @@ -1418,6 +1470,46 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "clap" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + [[package]] name = "cms" version = "0.2.3" @@ -1430,6 +1522,12 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "compare" version = "0.0.6" @@ -3021,6 +3119,12 @@ dependencies = [ "serde", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.13.0" @@ -3631,6 +3735,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -6262,6 +6372,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "utxorpc-spec" version = "0.15.0" diff --git a/Cargo.toml b/Cargo.toml index f0533b62..79b19645 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ caryatid_module_clock = "0.12" caryatid_module_spy = "0.12" anyhow = "1.0" chrono = "0.4" +clap = { version = "4.5", features = ["derive"] } config = "0.15.11" dashmap = "6.1.0" hex = "0.4" diff --git a/processes/omnibus/Cargo.toml b/processes/omnibus/Cargo.toml index 52a81cf6..752d94a9 100644 --- a/processes/omnibus/Cargo.toml +++ b/processes/omnibus/Cargo.toml @@ -34,6 +34,7 @@ caryatid_module_rest_server = { workspace = true } caryatid_module_spy = { workspace = true } anyhow = { workspace = true } +clap = { workspace = true } config = { workspace = true } tracing = { workspace = true } tracing-subscriber = { version = "0.3.20", features = ["registry", "env-filter"] } diff --git a/processes/omnibus/src/main.rs b/processes/omnibus/src/main.rs index ecbebffe..9db03ac5 100644 --- a/processes/omnibus/src/main.rs +++ b/processes/omnibus/src/main.rs @@ -44,9 +44,18 @@ use tikv_jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; +#[derive(Debug, clap::Parser)] +#[command(name = "acropolis_process_omnibus")] +struct Args { + #[arg(long, value_name = "PATH", default_value_t = option_env!("ACROPOLIS_OMNIBUS_DEFAULT_CONFIG").unwrap_or("omnibus.toml").to_string())] + config: String, +} + /// Standard main #[tokio::main] pub async fn main() -> Result<()> { + let args = ::parse(); + // Standard logging using RUST_LOG for log levels default to INFO for events only let fmt_layer = fmt::layer() .with_filter(EnvFilter::from_default_env().add_directive(filter::LevelFilter::INFO.into())) @@ -76,7 +85,7 @@ pub async fn main() -> Result<()> { // Read the config let config = Arc::new( Config::builder() - .add_source(File::with_name("omnibus")) + .add_source(File::with_name(&args.config)) .add_source(Environment::with_prefix("ACROPOLIS")) .build() .unwrap(), diff --git a/processes/replayer/Cargo.toml b/processes/replayer/Cargo.toml index 3b7084f9..127b225e 100644 --- a/processes/replayer/Cargo.toml +++ b/processes/replayer/Cargo.toml @@ -30,6 +30,7 @@ caryatid_module_rest_server = { workspace = true } caryatid_module_spy = { workspace = true } anyhow = { workspace = true } +clap = { workspace = true } config = { workspace = true } tracing = { workspace = true } tracing-subscriber = { version = "0.3.20", features = ["registry", "env-filter"] } diff --git a/processes/replayer/src/main.rs b/processes/replayer/src/main.rs index 2f4e28ef..90702968 100644 --- a/processes/replayer/src/main.rs +++ b/processes/replayer/src/main.rs @@ -5,7 +5,7 @@ use anyhow::Result; use caryatid_process::Process; use caryatid_sdk::ModuleRegistry; use config::{Config, Environment, File}; -use std::{env, sync::Arc}; +use std::sync::Arc; use tracing::info; use tracing_subscriber::prelude::*; use tracing_subscriber::{filter, fmt, EnvFilter, Registry}; @@ -97,8 +97,31 @@ fn setup_governance_replay(process: &mut dyn ModuleRegistry) { Spy::::register(process); } +#[derive(Debug, clap::Parser)] +#[command( + name = "acropolis_process_omnibus", + group(clap::ArgGroup::new("mode").required(true).args(&["governance_collect", "governance_replay", "alonzo_governance_collect"])), +)] +struct Args { + #[arg(long, value_name = "PATH", default_value_t = option_env!("ACROPOLIS_REPLAYER_DEFAULT_CONFIG").unwrap_or("replayer.toml").to_string())] + config: String, + + // FIXME: typically, these should be real [`clap::Command`] commands, not + // flags, but @michalrus kept `--` for backwards compatibility. + #[arg(long)] + governance_collect: bool, + + #[arg(long)] + governance_replay: bool, + + #[arg(long)] + alonzo_governance_collect: bool, +} + #[tokio::main] pub async fn main() -> Result<()> { + let args = ::parse(); + // Initialise tracing let fmt_layer = fmt::layer() .with_filter(EnvFilter::from_default_env().add_directive(filter::LevelFilter::INFO.into())) @@ -108,13 +131,10 @@ pub async fn main() -> Result<()> { info!("Acropolis omnibus process"); - let mut args = env::args(); - let _executable_name = args.next(); - // Read the config let config = Arc::new( Config::builder() - .add_source(File::with_name("replayer")) + .add_source(File::with_name(&args.config)) .add_source(Environment::with_prefix("ACROPOLIS")) .build() .unwrap(), @@ -123,22 +143,14 @@ pub async fn main() -> Result<()> { // Create the process let mut process = Process::::create(config).await; - if let Some(key) = args.next() { - match key.as_str() { - "--governance-collect" => setup_governance_collect(&mut process), - "--governance-replay" => setup_governance_replay(&mut process), - "--alonzo-governance-collect" => setup_alonzo_governance_collect(&mut process), - a => { - tracing::error!( - "Unknown command line argument: {a}, \ - expected --governance-collect or --governance-replay" - ); - return Ok(()); - } - } + if args.governance_collect { + setup_governance_collect(&mut process) + } else if args.governance_replay { + setup_governance_replay(&mut process) + } else if args.alonzo_governance_collect { + setup_alonzo_governance_collect(&mut process) } else { - tracing::error!("Please, specify command: command line must have at least one argument"); - return Ok(()); + unreachable!() } // Run it From 196219873bdc70c6b6ba73e3356bc1fa46403188 Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Mon, 20 Oct 2025 16:43:03 +0200 Subject: [PATCH 3/4] Fix the replayer process name in `clap` options --- processes/replayer/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processes/replayer/src/main.rs b/processes/replayer/src/main.rs index 90702968..a783829d 100644 --- a/processes/replayer/src/main.rs +++ b/processes/replayer/src/main.rs @@ -99,7 +99,7 @@ fn setup_governance_replay(process: &mut dyn ModuleRegistry) { #[derive(Debug, clap::Parser)] #[command( - name = "acropolis_process_omnibus", + name = "acropolis_process_replayer", group(clap::ArgGroup::new("mode").required(true).args(&["governance_collect", "governance_replay", "alonzo_governance_collect"])), )] struct Args { From 976793e75c47334176f83ad6f4fa7e6213c6ccbb Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Tue, 21 Oct 2025 14:48:13 +0200 Subject: [PATCH 4/4] Make `ACROPOLIS_OFFLINE_MIRROR` reference file paths, not contents --- modules/genesis_bootstrapper/build.rs | 6 ++++-- modules/parameters_state/build.rs | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/genesis_bootstrapper/build.rs b/modules/genesis_bootstrapper/build.rs index 9c6fa3f5..52886ce5 100644 --- a/modules/genesis_bootstrapper/build.rs +++ b/modules/genesis_bootstrapper/build.rs @@ -15,8 +15,10 @@ async fn fetch_bytes(client: &reqwest::Client, url: &str) -> Result> { if let Ok(path) = env::var("ACROPOLIS_OFFLINE_MIRROR") { if let Ok(file) = File::open(&path) { if let Ok(map) = from_reader::<_, HashMap>(file) { - if let Some(s) = map.get(url) { - return Ok(s.as_bytes().to_vec()); + if let Some(path) = map.get(url) { + if let Ok(bytes) = fs::read(&Path::new(path).to_path_buf()) { + return Ok(bytes); + } } } } diff --git a/modules/parameters_state/build.rs b/modules/parameters_state/build.rs index feac6e77..a380b618 100644 --- a/modules/parameters_state/build.rs +++ b/modules/parameters_state/build.rs @@ -14,9 +14,12 @@ fn fetch_text(url: &str) -> Result> { if let Ok(path) = env::var("ACROPOLIS_OFFLINE_MIRROR") { if !path.is_empty() { if let Ok(file) = File::open(path) { - let map: HashMap = from_reader(file)?; - if let Some(s) = map.get(url) { - return Ok(s.clone()); + if let Ok(map) = from_reader::<_, HashMap>(file) { + if let Some(path_str) = map.get(url) { + if let Ok(s) = fs::read_to_string(&Path::new(path_str).to_path_buf()) { + return Ok(s); + } + } } } }