diff --git a/Cargo.lock b/Cargo.lock index 8113613e..6887ef91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,6 +389,7 @@ dependencies = [ "caryatid_process", "caryatid_sdk", "chrono", + "clap", "config", "opentelemetry", "opentelemetry-otlp", @@ -428,6 +429,7 @@ dependencies = [ "caryatid_process", "caryatid_sdk", "chrono", + "clap", "config", "serde", "serde_json", @@ -589,6 +591,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", +] + [[package]] name = "anyhow" version = "1.0.100" @@ -1418,6 +1470,46 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "clap" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + [[package]] name = "cms" version = "0.2.3" @@ -1430,6 +1522,12 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "compare" version = "0.0.6" @@ -3021,6 +3119,12 @@ dependencies = [ "serde", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.13.0" @@ -3631,6 +3735,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -6262,6 +6372,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "utxorpc-spec" version = "0.15.0" diff --git a/Cargo.toml b/Cargo.toml index f0533b62..79b19645 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ caryatid_module_clock = "0.12" caryatid_module_spy = "0.12" anyhow = "1.0" chrono = "0.4" +clap = { version = "4.5", features = ["derive"] } config = "0.15.11" dashmap = "6.1.0" hex = "0.4" diff --git a/modules/genesis_bootstrapper/Cargo.toml b/modules/genesis_bootstrapper/Cargo.toml index 17fb440e..67a8f856 100644 --- a/modules/genesis_bootstrapper/Cargo.toml +++ b/modules/genesis_bootstrapper/Cargo.toml @@ -26,6 +26,7 @@ tracing = { workspace = true } [build-dependencies] anyhow = { workspace = true } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +serde_json = { workspace = true } tokio = { workspace = true } [lib] diff --git a/modules/genesis_bootstrapper/build.rs b/modules/genesis_bootstrapper/build.rs index 86bce1c5..52886ce5 100644 --- a/modules/genesis_bootstrapper/build.rs +++ b/modules/genesis_bootstrapper/build.rs @@ -1,18 +1,36 @@ // Build-time script to download generics +use std::collections::HashMap; +use std::env; use std::fs; +use std::fs::File; use std::io::Write; use std::path::Path; use anyhow::{Context, Result}; +use serde_json::from_reader; const OUTPUT_DIR: &str = "downloads"; +async fn fetch_bytes(client: &reqwest::Client, url: &str) -> Result> { + if let Ok(path) = env::var("ACROPOLIS_OFFLINE_MIRROR") { + if let Ok(file) = File::open(&path) { + if let Ok(map) = from_reader::<_, HashMap>(file) { + if let Some(path) = map.get(url) { + if let Ok(bytes) = fs::read(&Path::new(path).to_path_buf()) { + return Ok(bytes); + } + } + } + } + } + let req = client.get(url).build().with_context(|| format!("Failed to request {url}"))?; + let resp = client.execute(req).await.with_context(|| format!("Failed to fetch {url}"))?; + Ok(resp.bytes().await.context("Failed to read response")?.to_vec()) +} + /// Download a URL to a file in OUTPUT_DIR async fn download(client: &reqwest::Client, url: &str, filename: &str) -> Result<()> { - let request = client.get(url).build().with_context(|| format!("Failed to request {url}"))?; - let response = - client.execute(request).await.with_context(|| format!("Failed to fetch {url}"))?; - let data = response.bytes().await.context("Failed to read response")?; + let data = fetch_bytes(client, url).await?; let output_path = Path::new(OUTPUT_DIR); if !output_path.exists() { diff --git a/modules/parameters_state/Cargo.toml b/modules/parameters_state/Cargo.toml index 3b36030e..abeb9516 100644 --- a/modules/parameters_state/Cargo.toml +++ b/modules/parameters_state/Cargo.toml @@ -29,6 +29,7 @@ tracing = { workspace = true } [build-dependencies] reqwest = { version = "0.11", features = ["blocking"] } +serde_json = { workspace = true } [lib] path = "src/parameters_state.rs" diff --git a/modules/parameters_state/build.rs b/modules/parameters_state/build.rs index b6bb61dd..a380b618 100644 --- a/modules/parameters_state/build.rs +++ b/modules/parameters_state/build.rs @@ -1,16 +1,36 @@ // Build-time script to download generics use reqwest::blocking::get; +use serde_json::from_reader; +use std::collections::HashMap; +use std::env; use std::fs; +use std::fs::File; use std::io::Write; use std::path::Path; const OUTPUT_DIR: &str = "downloads"; +fn fetch_text(url: &str) -> Result> { + if let Ok(path) = env::var("ACROPOLIS_OFFLINE_MIRROR") { + if !path.is_empty() { + if let Ok(file) = File::open(path) { + if let Ok(map) = from_reader::<_, HashMap>(file) { + if let Some(path_str) = map.get(url) { + if let Ok(s) = fs::read_to_string(&Path::new(path_str).to_path_buf()) { + return Ok(s); + } + } + } + } + } + } + Ok(get(url)?.error_for_status()?.text()?) +} + /// Download a URL to a file in OUTPUT_DIR fn download(url_base: &str, epoch: &str, filename: &str, rename: &Vec<(&str, &str)>) { let url = format!("{}/{}-genesis.json", url_base, epoch); - let response = get(url).expect("Failed to fetch {url}"); - let mut data = response.text().expect("Failed to read response"); + let mut data = fetch_text(&url).expect("Failed to fetch {url}"); for (what, with) in rename.iter() { data = data.replace(&format!("\"{what}\""), &format!("\"{with}\"")); diff --git a/processes/omnibus/Cargo.toml b/processes/omnibus/Cargo.toml index 52a81cf6..752d94a9 100644 --- a/processes/omnibus/Cargo.toml +++ b/processes/omnibus/Cargo.toml @@ -34,6 +34,7 @@ caryatid_module_rest_server = { workspace = true } caryatid_module_spy = { workspace = true } anyhow = { workspace = true } +clap = { workspace = true } config = { workspace = true } tracing = { workspace = true } tracing-subscriber = { version = "0.3.20", features = ["registry", "env-filter"] } diff --git a/processes/omnibus/src/main.rs b/processes/omnibus/src/main.rs index ecbebffe..9db03ac5 100644 --- a/processes/omnibus/src/main.rs +++ b/processes/omnibus/src/main.rs @@ -44,9 +44,18 @@ use tikv_jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; +#[derive(Debug, clap::Parser)] +#[command(name = "acropolis_process_omnibus")] +struct Args { + #[arg(long, value_name = "PATH", default_value_t = option_env!("ACROPOLIS_OMNIBUS_DEFAULT_CONFIG").unwrap_or("omnibus.toml").to_string())] + config: String, +} + /// Standard main #[tokio::main] pub async fn main() -> Result<()> { + let args = ::parse(); + // Standard logging using RUST_LOG for log levels default to INFO for events only let fmt_layer = fmt::layer() .with_filter(EnvFilter::from_default_env().add_directive(filter::LevelFilter::INFO.into())) @@ -76,7 +85,7 @@ pub async fn main() -> Result<()> { // Read the config let config = Arc::new( Config::builder() - .add_source(File::with_name("omnibus")) + .add_source(File::with_name(&args.config)) .add_source(Environment::with_prefix("ACROPOLIS")) .build() .unwrap(), diff --git a/processes/replayer/Cargo.toml b/processes/replayer/Cargo.toml index 3b7084f9..127b225e 100644 --- a/processes/replayer/Cargo.toml +++ b/processes/replayer/Cargo.toml @@ -30,6 +30,7 @@ caryatid_module_rest_server = { workspace = true } caryatid_module_spy = { workspace = true } anyhow = { workspace = true } +clap = { workspace = true } config = { workspace = true } tracing = { workspace = true } tracing-subscriber = { version = "0.3.20", features = ["registry", "env-filter"] } diff --git a/processes/replayer/src/main.rs b/processes/replayer/src/main.rs index 2f4e28ef..a783829d 100644 --- a/processes/replayer/src/main.rs +++ b/processes/replayer/src/main.rs @@ -5,7 +5,7 @@ use anyhow::Result; use caryatid_process::Process; use caryatid_sdk::ModuleRegistry; use config::{Config, Environment, File}; -use std::{env, sync::Arc}; +use std::sync::Arc; use tracing::info; use tracing_subscriber::prelude::*; use tracing_subscriber::{filter, fmt, EnvFilter, Registry}; @@ -97,8 +97,31 @@ fn setup_governance_replay(process: &mut dyn ModuleRegistry) { Spy::::register(process); } +#[derive(Debug, clap::Parser)] +#[command( + name = "acropolis_process_replayer", + group(clap::ArgGroup::new("mode").required(true).args(&["governance_collect", "governance_replay", "alonzo_governance_collect"])), +)] +struct Args { + #[arg(long, value_name = "PATH", default_value_t = option_env!("ACROPOLIS_REPLAYER_DEFAULT_CONFIG").unwrap_or("replayer.toml").to_string())] + config: String, + + // FIXME: typically, these should be real [`clap::Command`] commands, not + // flags, but @michalrus kept `--` for backwards compatibility. + #[arg(long)] + governance_collect: bool, + + #[arg(long)] + governance_replay: bool, + + #[arg(long)] + alonzo_governance_collect: bool, +} + #[tokio::main] pub async fn main() -> Result<()> { + let args = ::parse(); + // Initialise tracing let fmt_layer = fmt::layer() .with_filter(EnvFilter::from_default_env().add_directive(filter::LevelFilter::INFO.into())) @@ -108,13 +131,10 @@ pub async fn main() -> Result<()> { info!("Acropolis omnibus process"); - let mut args = env::args(); - let _executable_name = args.next(); - // Read the config let config = Arc::new( Config::builder() - .add_source(File::with_name("replayer")) + .add_source(File::with_name(&args.config)) .add_source(Environment::with_prefix("ACROPOLIS")) .build() .unwrap(), @@ -123,22 +143,14 @@ pub async fn main() -> Result<()> { // Create the process let mut process = Process::::create(config).await; - if let Some(key) = args.next() { - match key.as_str() { - "--governance-collect" => setup_governance_collect(&mut process), - "--governance-replay" => setup_governance_replay(&mut process), - "--alonzo-governance-collect" => setup_alonzo_governance_collect(&mut process), - a => { - tracing::error!( - "Unknown command line argument: {a}, \ - expected --governance-collect or --governance-replay" - ); - return Ok(()); - } - } + if args.governance_collect { + setup_governance_collect(&mut process) + } else if args.governance_replay { + setup_governance_replay(&mut process) + } else if args.alonzo_governance_collect { + setup_alonzo_governance_collect(&mut process) } else { - tracing::error!("Please, specify command: command line must have at least one argument"); - return Ok(()); + unreachable!() } // Run it