From 3bf6e0a46d52ab99f5cfd73fb01a1b51badb2535 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 19 Mar 2025 13:11:27 +0100 Subject: [PATCH 01/13] Prefactor: Update bitcoind used in CI to v27.2 .. we need to bump the version, while making sure `electrsd` and CLN CI are using the same version. As the `blockstream/bitcoind` version hasn't reached v28 yet, we opt for v27.2 everywhere for now. --- docker-compose-cln.yml | 4 +--- scripts/download_bitcoind_electrs.sh | 12 ++++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docker-compose-cln.yml b/docker-compose-cln.yml index 5fb1f2dcd..7978bea35 100644 --- a/docker-compose-cln.yml +++ b/docker-compose-cln.yml @@ -1,8 +1,6 @@ -version: '3' - services: bitcoin: - image: blockstream/bitcoind:24.1 + image: blockstream/bitcoind:27.2 platform: linux/amd64 command: [ diff --git a/scripts/download_bitcoind_electrs.sh b/scripts/download_bitcoind_electrs.sh index fcdb31891..47a95332e 100755 --- a/scripts/download_bitcoind_electrs.sh +++ b/scripts/download_bitcoind_electrs.sh @@ -1,3 +1,6 @@ +#!/bin/bash +set -eox pipefail + # Our Esplora-based tests require `electrs` and `bitcoind` # binaries. Here, we download the binaries, validate them, and export their # location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the @@ -7,19 +10,20 @@ HOST_PLATFORM="$(rustc --version --verbose | grep "host:" | awk '{ print $2 }')" ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases" ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254" BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/" -BITCOIND_VERSION="25.1" +BITCOIND_VERSION="27.2" if [[ "$HOST_PLATFORM" == *linux* ]]; then ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1" BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz - BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b" + BITCOIND_DL_HASH="acc223af46c178064c132b235392476f66d486453ddbd6bca6f1f8411547da78" elif [[ "$HOST_PLATFORM" == *darwin* ]]; then ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd" BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz - BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1" + BITCOIND_DL_HASH="6ebc56ca1397615d5a6df2b5cf6727b768e3dcac320c2d5c2f321dcaabc7efa2" else - echo "\n\nUnsupported platform: $HOST_PLATFORM Exiting.." + printf "\n\n" + echo "Unsupported platform: $HOST_PLATFORM Exiting.." exit 1 fi From df47d33472b2e759a4226e1bad3406759a31c68d Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 3 Feb 2025 13:48:56 +0100 Subject: [PATCH 02/13] Prefactor: Upgrade to `electrum-client` v0.22 We upgrade our tests to use `electrum-client` v0.22 and `electrsd` v0.31 to ensure compatibility with `bdk_electrum` were about to start using. --- Cargo.toml | 8 ++++---- tests/common/mod.rs | 30 ++++++++++++------------------ tests/integration_tests_cln.rs | 8 ++++---- tests/integration_tests_lnd.rs | 8 ++++---- tests/integration_tests_rust.rs | 32 ++++++++++---------------------- 5 files changed, 34 insertions(+), 52 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5481c76dd..43d6bc872 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,16 +92,16 @@ winapi = { version = "0.3", features = ["winbase"] } lightning = { version = "0.1.0", features = ["std", "_test_utils"] } #lightning = { git = "https://github.com/lightningdevkit/rust-lightning", branch="main", features = ["std", "_test_utils"] } #lightning = { path = "../rust-lightning/lightning", features = ["std", "_test_utils"] } -electrum-client = { version = "0.21.0", default-features = true } -bitcoincore-rpc = { version = "0.19.0", default-features = false } +electrum-client = { version = "0.22.0", default-features = true } proptest = "1.0.0" regex = "1.5.6" [target.'cfg(not(no_download))'.dev-dependencies] -electrsd = { version = "0.29.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] } +electrsd = { version = "0.33.0", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] } [target.'cfg(no_download)'.dev-dependencies] -electrsd = { version = "0.29.0", features = ["legacy"] } +electrsd = { version = "0.33.0", default-features = false, features = ["legacy"] } +corepc-node = { version = "0.7.0", default-features = false, features = ["27_2"] } [target.'cfg(cln_test)'.dev-dependencies] clightningrpc = { version = "0.3.0-beta.8", default-features = false } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 2a3e99c0f..ac8d1ce90 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -33,11 +33,9 @@ use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::hashes::Hash; use bitcoin::{Address, Amount, Network, OutPoint, Txid}; -use bitcoincore_rpc::bitcoincore_rpc_json::AddressType; -use bitcoincore_rpc::Client as BitcoindClient; -use bitcoincore_rpc::RpcApi; - -use electrsd::{bitcoind, bitcoind::BitcoinD, ElectrsD}; +use electrsd::corepc_node::Client as BitcoindClient; +use electrsd::corepc_node::Node as BitcoinD; +use electrsd::{corepc_node, ElectrsD}; use electrum_client::ElectrumApi; use rand::distributions::Alphanumeric; @@ -171,10 +169,10 @@ pub(crate) use expect_payment_successful_event; pub(crate) fn setup_bitcoind_and_electrsd() -> (BitcoinD, ElectrsD) { let bitcoind_exe = - env::var("BITCOIND_EXE").ok().or_else(|| bitcoind::downloaded_exe_path().ok()).expect( + env::var("BITCOIND_EXE").ok().or_else(|| corepc_node::downloaded_exe_path().ok()).expect( "you need to provide an env var BITCOIND_EXE or specify a bitcoind version feature", ); - let mut bitcoind_conf = bitcoind::Conf::default(); + let mut bitcoind_conf = corepc_node::Conf::default(); bitcoind_conf.network = "regtest"; let bitcoind = BitcoinD::with_conf(bitcoind_exe, &bitcoind_conf).unwrap(); @@ -359,17 +357,14 @@ pub(crate) fn setup_node( pub(crate) fn generate_blocks_and_wait( bitcoind: &BitcoindClient, electrs: &E, num: usize, ) { - let _ = bitcoind.create_wallet("ldk_node_test", None, None, None, None); + let _ = bitcoind.create_wallet("ldk_node_test"); let _ = bitcoind.load_wallet("ldk_node_test"); print!("Generating {} blocks...", num); - let cur_height = bitcoind.get_block_count().expect("failed to get current block height"); - let address = bitcoind - .get_new_address(Some("test"), Some(AddressType::Legacy)) - .expect("failed to get new address") - .require_network(bitcoin::Network::Regtest) - .expect("failed to get new address"); + let blockchain_info = bitcoind.get_blockchain_info().expect("failed to get blockchain info"); + let cur_height = blockchain_info.blocks; + let address = bitcoind.new_address().expect("failed to get new address"); // TODO: expect this Result once the WouldBlock issue is resolved upstream. - let _block_hashes_res = bitcoind.generate_to_address(num as u64, &address); + let _block_hashes_res = bitcoind.generate_to_address(num, &address); wait_for_block(electrs, cur_height as usize + num); print!(" Done!"); println!("\n"); @@ -450,13 +445,12 @@ where pub(crate) fn premine_and_distribute_funds( bitcoind: &BitcoindClient, electrs: &E, addrs: Vec
, amount: Amount, ) { - let _ = bitcoind.create_wallet("ldk_node_test", None, None, None, None); + let _ = bitcoind.create_wallet("ldk_node_test"); let _ = bitcoind.load_wallet("ldk_node_test"); generate_blocks_and_wait(bitcoind, electrs, 101); for addr in addrs { - let txid = - bitcoind.send_to_address(&addr, amount, None, None, None, None, None, None).unwrap(); + let txid = bitcoind.send_to_address(&addr, amount).unwrap().0.parse().unwrap(); wait_for_tx(electrs, txid); } diff --git a/tests/integration_tests_cln.rs b/tests/integration_tests_cln.rs index 875922ce2..b6300576c 100644 --- a/tests/integration_tests_cln.rs +++ b/tests/integration_tests_cln.rs @@ -18,8 +18,8 @@ use lightning_invoice::{Bolt11InvoiceDescription, Description}; use clightningrpc::lightningrpc::LightningRPC; use clightningrpc::responses::NetworkAddress; -use bitcoincore_rpc::Auth; -use bitcoincore_rpc::Client as BitcoindClient; +use electrsd::corepc_client::client_sync::Auth; +use electrsd::corepc_node::Client as BitcoindClient; use electrum_client::Client as ElectrumClient; use lightning_invoice::Bolt11Invoice; @@ -33,8 +33,8 @@ use std::str::FromStr; #[test] fn test_cln() { // Setup bitcoind / electrs clients - let bitcoind_client = BitcoindClient::new( - "127.0.0.1:18443", + let bitcoind_client = BitcoindClient::new_with_auth( + "http://127.0.0.1:18443", Auth::UserPass("user".to_string(), "pass".to_string()), ) .unwrap(); diff --git a/tests/integration_tests_lnd.rs b/tests/integration_tests_lnd.rs index feb981d8e..0232e8f2e 100755 --- a/tests/integration_tests_lnd.rs +++ b/tests/integration_tests_lnd.rs @@ -15,8 +15,8 @@ use lnd_grpc_rust::lnrpc::{ }; use lnd_grpc_rust::{connect, LndClient}; -use bitcoincore_rpc::Auth; -use bitcoincore_rpc::Client as BitcoindClient; +use electrsd::corepc_client::client_sync::Auth; +use electrsd::corepc_node::Client as BitcoindClient; use electrum_client::Client as ElectrumClient; use lightning_invoice::{Bolt11InvoiceDescription, Description}; @@ -30,8 +30,8 @@ use tokio::fs; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_lnd() { // Setup bitcoind / electrs clients - let bitcoind_client = BitcoindClient::new( - "127.0.0.1:18443", + let bitcoind_client = BitcoindClient::new_with_auth( + "http://127.0.0.1:18443", Auth::UserPass("user".to_string(), "pass".to_string()), ) .unwrap(); diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index da9eea48f..9bbd56689 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -28,11 +28,11 @@ use lightning::ln::channelmanager::PaymentId; use lightning::routing::gossip::{NodeAlias, NodeId}; use lightning::util::persist::KVStore; -use bitcoincore_rpc::RpcApi; +use lightning_invoice::{Bolt11InvoiceDescription, Description}; use bitcoin::hashes::Hash; use bitcoin::Amount; -use lightning_invoice::{Bolt11InvoiceDescription, Description}; + use log::LevelFilter; use std::sync::Arc; @@ -500,16 +500,10 @@ fn onchain_wallet_recovery() { let txid = bitcoind .client - .send_to_address( - &addr_2, - Amount::from_sat(premine_amount_sat), - None, - None, - None, - None, - None, - None, - ) + .send_to_address(&addr_2, Amount::from_sat(premine_amount_sat)) + .unwrap() + .0 + .parse() .unwrap(); wait_for_tx(&electrsd.client, txid); @@ -542,16 +536,10 @@ fn onchain_wallet_recovery() { let txid = bitcoind .client - .send_to_address( - &addr_6, - Amount::from_sat(premine_amount_sat), - None, - None, - None, - None, - None, - None, - ) + .send_to_address(&addr_6, Amount::from_sat(premine_amount_sat)) + .unwrap() + .0 + .parse() .unwrap(); wait_for_tx(&electrsd.client, txid); From 2187046e90e404516163adcde83b1fd4c7705ce4 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Tue, 18 Mar 2025 12:23:19 +0100 Subject: [PATCH 03/13] Install `bindgen-cli` in Kotlin CI By default `rustls`, our TLS implementation of choice, uses `aws-lc-rs` which requires `bindgen` on some platforms. To allow building with `aws-lc-rs` on Android, we here install the `bindgen-cli` tool before running the bindings generation script in CI. --- .github/workflows/kotlin.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/kotlin.yml b/.github/workflows/kotlin.yml index a1711ba49..5cb1b8c27 100644 --- a/.github/workflows/kotlin.yml +++ b/.github/workflows/kotlin.yml @@ -39,6 +39,9 @@ jobs: - name: Generate Kotlin JVM run: ./scripts/uniffi_bindgen_generate_kotlin.sh + - name: Install `bindgen-cli` + run: cargo install --force bindgen-cli + - name: Generate Kotlin Android run: ./scripts/uniffi_bindgen_generate_kotlin_android.sh From 274a21ff5d22187076c61cc0d8175adb66c53439 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Tue, 4 Feb 2025 14:45:34 +0100 Subject: [PATCH 04/13] Add `bdk_electrum`/`transaction-sync` `electrum` support in `Cargo.toml` --- Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 43d6bc872..5166a3ec5 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ lightning-persister = { version = "0.1.0" } lightning-background-processor = { version = "0.1.0", features = ["futures"] } lightning-rapid-gossip-sync = { version = "0.1.0" } lightning-block-sync = { version = "0.1.0", features = ["rpc-client", "tokio"] } -lightning-transaction-sync = { version = "0.1.0", features = ["esplora-async-https", "time"] } +lightning-transaction-sync = { version = "0.1.0", features = ["esplora-async-https", "time", "electrum"] } lightning-liquidity = { version = "0.1.0", features = ["std"] } #lightning = { git = "https://github.com/lightningdevkit/rust-lightning", branch = "main", features = ["std"] } @@ -63,6 +63,7 @@ lightning-liquidity = { version = "0.1.0", features = ["std"] } bdk_chain = { version = "0.21.1", default-features = false, features = ["std"] } bdk_esplora = { version = "0.20.1", default-features = false, features = ["async-https-rustls", "tokio"]} +bdk_electrum = { version = "0.20.1", default-features = false, features = ["use-rustls"]} bdk_wallet = { version = "1.0.0", default-features = false, features = ["std", "keys-bip39"]} reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] } @@ -76,6 +77,7 @@ rand = "0.8.5" chrono = { version = "0.4", default-features = false, features = ["clock"] } tokio = { version = "1.37", default-features = false, features = [ "rt-multi-thread", "time", "sync", "macros" ] } esplora-client = { version = "0.11", default-features = false, features = ["tokio", "async-https-rustls"] } +electrum-client = { version = "0.22.0", default-features = true } libc = "0.2" uniffi = { version = "0.27.3", features = ["build"], optional = true } serde = { version = "1.0.210", default-features = false, features = ["std", "derive"] } @@ -92,7 +94,6 @@ winapi = { version = "0.3", features = ["winbase"] } lightning = { version = "0.1.0", features = ["std", "_test_utils"] } #lightning = { git = "https://github.com/lightningdevkit/rust-lightning", branch="main", features = ["std", "_test_utils"] } #lightning = { path = "../rust-lightning/lightning", features = ["std", "_test_utils"] } -electrum-client = { version = "0.22.0", default-features = true } proptest = "1.0.0" regex = "1.5.6" From e793b6f23128384916d0b0212bd3cf66bf6e568c Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 7 Mar 2025 13:45:23 +0100 Subject: [PATCH 05/13] Allow to configure `ChainSource::Electrum` via builder We here setup the basic API and structure for the `ChainSource::Electrum`. Currently, we won't have a `Runtime` available when initializing `ChainSource::Electrum` in `Builder::build`. We therefore isolate any runtime-specific behavior into an `ElectrumRuntimeClient`. This might change in the future, but for now we do need this workaround. --- bindings/ldk_node.udl | 5 ++ src/builder.rs | 41 +++++++++++++- src/chain/electrum.rs | 60 +++++++++++++++++++++ src/chain/mod.rs | 121 +++++++++++++++++++++++++++++++++++++++++- src/config.rs | 23 +++++++- src/lib.rs | 16 ++++++ src/uniffi_types.rs | 4 +- 7 files changed, 264 insertions(+), 6 deletions(-) create mode 100644 src/chain/electrum.rs diff --git a/bindings/ldk_node.udl b/bindings/ldk_node.udl index 1e4526347..1a56a7d4b 100644 --- a/bindings/ldk_node.udl +++ b/bindings/ldk_node.udl @@ -30,6 +30,10 @@ dictionary EsploraSyncConfig { BackgroundSyncConfig? background_sync_config; }; +dictionary ElectrumSyncConfig { + BackgroundSyncConfig? background_sync_config; +}; + dictionary LSPS2ServiceConfig { string? require_token; boolean advertise_service; @@ -72,6 +76,7 @@ interface Builder { void set_entropy_seed_bytes(sequence seed_bytes); void set_entropy_bip39_mnemonic(Mnemonic mnemonic, string? passphrase); void set_chain_source_esplora(string server_url, EsploraSyncConfig? config); + void set_chain_source_electrum(string server_url, ElectrumSyncConfig? config); void set_chain_source_bitcoind_rpc(string rpc_host, u16 rpc_port, string rpc_user, string rpc_password); void set_gossip_source_p2p(); void set_gossip_source_rgs(string rgs_server_url); diff --git a/src/builder.rs b/src/builder.rs index 3e8fd2cba..224cc9fa7 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -7,8 +7,8 @@ use crate::chain::{ChainSource, DEFAULT_ESPLORA_SERVER_URL}; use crate::config::{ - default_user_config, may_announce_channel, AnnounceError, Config, EsploraSyncConfig, - DEFAULT_LOG_FILENAME, DEFAULT_LOG_LEVEL, WALLET_KEYS_SEED_LEN, + default_user_config, may_announce_channel, AnnounceError, Config, ElectrumSyncConfig, + EsploraSyncConfig, DEFAULT_LOG_FILENAME, DEFAULT_LOG_LEVEL, WALLET_KEYS_SEED_LEN, }; use crate::connection::ConnectionManager; @@ -83,6 +83,7 @@ const LSPS_HARDENED_CHILD_INDEX: u32 = 577; #[derive(Debug, Clone)] enum ChainDataSourceConfig { Esplora { server_url: String, sync_config: Option }, + Electrum { server_url: String, sync_config: Option }, BitcoindRpc { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String }, } @@ -284,6 +285,18 @@ impl NodeBuilder { self } + /// Configures the [`Node`] instance to source its chain data from the given Electrum server. + /// + /// If no `sync_config` is given, default values are used. See [`ElectrumSyncConfig`] for more + /// information. + pub fn set_chain_source_electrum( + &mut self, server_url: String, sync_config: Option, + ) -> &mut Self { + self.chain_data_source_config = + Some(ChainDataSourceConfig::Electrum { server_url, sync_config }); + self + } + /// Configures the [`Node`] instance to source its chain data from the given Bitcoin Core RPC /// endpoint. pub fn set_chain_source_bitcoind_rpc( @@ -691,6 +704,16 @@ impl ArcedNodeBuilder { self.inner.write().unwrap().set_chain_source_esplora(server_url, sync_config); } + /// Configures the [`Node`] instance to source its chain data from the given Electrum server. + /// + /// If no `sync_config` is given, default values are used. See [`ElectrumSyncConfig`] for more + /// information. + pub fn set_chain_source_electrum( + &self, server_url: String, sync_config: Option, + ) { + self.inner.write().unwrap().set_chain_source_electrum(server_url, sync_config); + } + /// Configures the [`Node`] instance to source its chain data from the given Bitcoin Core RPC /// endpoint. pub fn set_chain_source_bitcoind_rpc( @@ -1024,6 +1047,20 @@ fn build_with_store_internal( Arc::clone(&node_metrics), )) }, + Some(ChainDataSourceConfig::Electrum { server_url, sync_config }) => { + let sync_config = sync_config.unwrap_or(ElectrumSyncConfig::default()); + Arc::new(ChainSource::new_electrum( + server_url.clone(), + sync_config, + Arc::clone(&wallet), + Arc::clone(&fee_estimator), + Arc::clone(&tx_broadcaster), + Arc::clone(&kv_store), + Arc::clone(&config), + Arc::clone(&logger), + Arc::clone(&node_metrics), + )) + }, Some(ChainDataSourceConfig::BitcoindRpc { rpc_host, rpc_port, rpc_user, rpc_password }) => { Arc::new(ChainSource::new_bitcoind_rpc( rpc_host.clone(), diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs new file mode 100644 index 000000000..4e62a75a1 --- /dev/null +++ b/src/chain/electrum.rs @@ -0,0 +1,60 @@ +// This file is Copyright its original authors, visible in version control history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license , at your option. You may not use this file except in +// accordance with one or both of these licenses. + +use crate::error::Error; +use crate::logger::{log_error, LdkLogger, Logger}; + +use lightning_transaction_sync::ElectrumSyncClient; + +use bdk_electrum::BdkElectrumClient; + +use electrum_client::Client as ElectrumClient; +use electrum_client::ConfigBuilder as ElectrumConfigBuilder; + +use std::sync::Arc; + +const ELECTRUM_CLIENT_NUM_RETRIES: u8 = 3; +const ELECTRUM_CLIENT_TIMEOUT_SECS: u8 = 20; + +pub(crate) struct ElectrumRuntimeClient { + electrum_client: Arc, + bdk_electrum_client: Arc>, + tx_sync: Arc>>, + runtime: Arc, + logger: Arc, +} + +impl ElectrumRuntimeClient { + pub(crate) fn new( + server_url: String, runtime: Arc, logger: Arc, + ) -> Result { + let electrum_config = ElectrumConfigBuilder::new() + .retry(ELECTRUM_CLIENT_NUM_RETRIES) + .timeout(Some(ELECTRUM_CLIENT_TIMEOUT_SECS)) + .build(); + + let electrum_client = Arc::new( + ElectrumClient::from_config(&server_url, electrum_config.clone()).map_err(|e| { + log_error!(logger, "Failed to connect to electrum server: {}", e); + Error::ConnectionFailed + })?, + ); + let electrum_client_2 = + ElectrumClient::from_config(&server_url, electrum_config).map_err(|e| { + log_error!(logger, "Failed to connect to electrum server: {}", e); + Error::ConnectionFailed + })?; + let bdk_electrum_client = Arc::new(BdkElectrumClient::new(electrum_client_2)); + let tx_sync = Arc::new( + ElectrumSyncClient::new(server_url.clone(), Arc::clone(&logger)).map_err(|e| { + log_error!(logger, "Failed to connect to electrum server: {}", e); + Error::ConnectionFailed + })?, + ); + Ok(Self { electrum_client, bdk_electrum_client, tx_sync, runtime, logger }) + } +} diff --git a/src/chain/mod.rs b/src/chain/mod.rs index a2e01884a..ad213405a 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -6,12 +6,14 @@ // accordance with one or both of these licenses. mod bitcoind_rpc; +mod electrum; use crate::chain::bitcoind_rpc::{ BitcoindRpcClient, BoundedHeaderCache, ChainListener, FeeRateEstimationMode, }; +use crate::chain::electrum::ElectrumRuntimeClient; use crate::config::{ - Config, EsploraSyncConfig, BDK_CLIENT_CONCURRENCY, BDK_CLIENT_STOP_GAP, + Config, ElectrumSyncConfig, EsploraSyncConfig, BDK_CLIENT_CONCURRENCY, BDK_CLIENT_STOP_GAP, BDK_WALLET_SYNC_TIMEOUT_SECS, FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS, LDK_WALLET_SYNC_TIMEOUT_SECS, RESOLVED_CHANNEL_MONITOR_ARCHIVAL_INTERVAL, TX_BROADCAST_TIMEOUT_SECS, WALLET_SYNC_INTERVAL_MINIMUM_SECS, @@ -107,6 +109,45 @@ impl WalletSyncStatus { } } +pub(crate) enum ElectrumRuntimeStatus { + Started(Arc), + Stopped, +} + +impl ElectrumRuntimeStatus { + pub(crate) fn new() -> Self { + Self::Stopped + } + + pub(crate) fn start( + &mut self, server_url: String, runtime: Arc, logger: Arc, + ) -> Result<(), Error> { + match self { + Self::Stopped => { + let client = + Arc::new(ElectrumRuntimeClient::new(server_url.clone(), runtime, logger)?); + + *self = Self::Started(client); + }, + Self::Started(_) => { + debug_assert!(false, "We shouldn't call start if we're already started") + }, + } + Ok(()) + } + + pub(crate) fn stop(&mut self) { + *self = Self::new() + } + + pub(crate) fn client(&self) -> Option<&Arc> { + match self { + Self::Started(client) => Some(&client), + Self::Stopped { .. } => None, + } + } +} + pub(crate) enum ChainSource { Esplora { sync_config: EsploraSyncConfig, @@ -122,6 +163,20 @@ pub(crate) enum ChainSource { logger: Arc, node_metrics: Arc>, }, + Electrum { + server_url: String, + sync_config: ElectrumSyncConfig, + electrum_runtime_status: RwLock, + onchain_wallet: Arc, + onchain_wallet_sync_status: Mutex, + lightning_wallet_sync_status: Mutex, + fee_estimator: Arc, + tx_broadcaster: Arc, + kv_store: Arc, + config: Arc, + logger: Arc, + node_metrics: Arc>, + }, BitcoindRpc { bitcoind_rpc_client: Arc, header_cache: tokio::sync::Mutex, @@ -167,6 +222,31 @@ impl ChainSource { } } + pub(crate) fn new_electrum( + server_url: String, sync_config: ElectrumSyncConfig, onchain_wallet: Arc, + fee_estimator: Arc, tx_broadcaster: Arc, + kv_store: Arc, config: Arc, logger: Arc, + node_metrics: Arc>, + ) -> Self { + let electrum_runtime_status = RwLock::new(ElectrumRuntimeStatus::new()); + let onchain_wallet_sync_status = Mutex::new(WalletSyncStatus::Completed); + let lightning_wallet_sync_status = Mutex::new(WalletSyncStatus::Completed); + Self::Electrum { + server_url, + sync_config, + electrum_runtime_status, + onchain_wallet, + onchain_wallet_sync_status, + lightning_wallet_sync_status, + fee_estimator, + tx_broadcaster, + kv_store, + config, + logger, + node_metrics, + } + } + pub(crate) fn new_bitcoind_rpc( host: String, port: u16, rpc_user: String, rpc_password: String, onchain_wallet: Arc, fee_estimator: Arc, @@ -193,6 +273,33 @@ impl ChainSource { } } + pub(crate) fn start(&self, runtime: Arc) -> Result<(), Error> { + match self { + Self::Electrum { server_url, electrum_runtime_status, logger, .. } => { + electrum_runtime_status.write().unwrap().start( + server_url.clone(), + runtime, + Arc::clone(&logger), + )?; + }, + _ => { + // Nothing to do for other chain sources. + }, + } + Ok(()) + } + + pub(crate) fn stop(&self) { + match self { + Self::Electrum { electrum_runtime_status, .. } => { + electrum_runtime_status.write().unwrap().stop(); + }, + _ => { + // Nothing to do for other chain sources. + }, + } + } + pub(crate) fn as_utxo_source(&self) -> Option> { match self { Self::BitcoindRpc { bitcoind_rpc_client, .. } => Some(bitcoind_rpc_client.rpc_client()), @@ -271,6 +378,7 @@ impl ChainSource { return; } }, + Self::Electrum { .. } => todo!(), Self::BitcoindRpc { bitcoind_rpc_client, header_cache, @@ -538,6 +646,7 @@ impl ChainSource { res }, + Self::Electrum { .. } => todo!(), Self::BitcoindRpc { .. } => { // In BitcoindRpc mode we sync lightning and onchain wallet in one go by via // `ChainPoller`. So nothing to do here. @@ -637,6 +746,7 @@ impl ChainSource { res }, + Self::Electrum { .. } => todo!(), Self::BitcoindRpc { .. } => { // In BitcoindRpc mode we sync lightning and onchain wallet in one go by via // `ChainPoller`. So nothing to do here. @@ -655,6 +765,11 @@ impl ChainSource { // `sync_onchain_wallet` and `sync_lightning_wallet`. So nothing to do here. unreachable!("Listeners will be synced via transction-based syncing") }, + Self::Electrum { .. } => { + // In Electrum mode we sync lightning and onchain wallets via + // `sync_onchain_wallet` and `sync_lightning_wallet`. So nothing to do here. + unreachable!("Listeners will be synced via transction-based syncing") + }, Self::BitcoindRpc { bitcoind_rpc_client, header_cache, @@ -875,6 +990,7 @@ impl ChainSource { Ok(()) }, + Self::Electrum { .. } => todo!(), Self::BitcoindRpc { bitcoind_rpc_client, fee_estimator, @@ -1085,6 +1201,7 @@ impl ChainSource { } } }, + Self::Electrum { .. } => todo!(), Self::BitcoindRpc { bitcoind_rpc_client, tx_broadcaster, logger, .. } => { // While it's a bit unclear when we'd be able to lean on Bitcoin Core >v28 // features, we should eventually switch to use `submitpackage` via the @@ -1147,12 +1264,14 @@ impl Filter for ChainSource { fn register_tx(&self, txid: &bitcoin::Txid, script_pubkey: &bitcoin::Script) { match self { Self::Esplora { tx_sync, .. } => tx_sync.register_tx(txid, script_pubkey), + Self::Electrum { .. } => todo!(), Self::BitcoindRpc { .. } => (), } } fn register_output(&self, output: lightning::chain::WatchedOutput) { match self { Self::Esplora { tx_sync, .. } => tx_sync.register_output(output), + Self::Electrum { .. } => todo!(), Self::BitcoindRpc { .. } => (), } } diff --git a/src/config.rs b/src/config.rs index 46b528488..4a39c1b56 100644 --- a/src/config.rs +++ b/src/config.rs @@ -364,7 +364,7 @@ pub struct EsploraSyncConfig { /// Background sync configuration. /// /// If set to `None`, background syncing will be disabled. Users will need to manually - /// sync via `Node::sync_wallets` for the wallets and fee rate updates. + /// sync via [`Node::sync_wallets`] for the wallets and fee rate updates. /// /// [`Node::sync_wallets`]: crate::Node::sync_wallets pub background_sync_config: Option, @@ -376,6 +376,27 @@ impl Default for EsploraSyncConfig { } } +/// Configuration for syncing with an Electrum backend. +/// +/// Background syncing is enabled by default, using the default values specified in +/// [`BackgroundSyncConfig`]. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct ElectrumSyncConfig { + /// Background sync configuration. + /// + /// If set to `None`, background syncing will be disabled. Users will need to manually + /// sync via [`Node::sync_wallets`] for the wallets and fee rate updates. + /// + /// [`Node::sync_wallets`]: crate::Node::sync_wallets + pub background_sync_config: Option, +} + +impl Default for ElectrumSyncConfig { + fn default() -> Self { + Self { background_sync_config: Some(BackgroundSyncConfig::default()) } + } +} + /// Options which apply on a per-channel basis and may change at runtime or based on negotiation /// with our counterparty. #[derive(Copy, Clone, Debug, PartialEq, Eq)] diff --git a/src/lib.rs b/src/lib.rs index a80db1e8c..93393585d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -236,6 +236,12 @@ impl Node { self.config.network ); + // Start up any runtime-dependant chain sources (e.g. Electrum) + self.chain_source.start(Arc::clone(&runtime)).map_err(|e| { + log_error!(self.logger, "Failed to start chain syncing: {}", e); + e + })?; + // Block to ensure we update our fee rate cache once on startup let chain_source = Arc::clone(&self.chain_source); let runtime_ref = &runtime; @@ -640,6 +646,9 @@ impl Node { log_info!(self.logger, "Shutting down LDK Node with node ID {}...", self.node_id()); + // Stop any runtime-dependant chain sources. + self.chain_source.stop(); + // Stop the runtime. match self.stop_sender.send(()) { Ok(_) => (), @@ -1257,6 +1266,13 @@ impl Node { .await?; chain_source.sync_onchain_wallet().await?; }, + ChainSource::Electrum { .. } => { + chain_source.update_fee_rate_estimates().await?; + chain_source + .sync_lightning_wallet(sync_cman, sync_cmon, sync_sweeper) + .await?; + chain_source.sync_onchain_wallet().await?; + }, ChainSource::BitcoindRpc { .. } => { chain_source.update_fee_rate_estimates().await?; chain_source diff --git a/src/uniffi_types.rs b/src/uniffi_types.rs index 410ce8531..acdee3a94 100644 --- a/src/uniffi_types.rs +++ b/src/uniffi_types.rs @@ -11,8 +11,8 @@ // Make sure to add any re-exported items that need to be used in uniffi below. pub use crate::config::{ - default_config, AnchorChannelsConfig, BackgroundSyncConfig, EsploraSyncConfig, - MaxDustHTLCExposure, + default_config, AnchorChannelsConfig, BackgroundSyncConfig, ElectrumSyncConfig, + EsploraSyncConfig, MaxDustHTLCExposure, }; pub use crate::graph::{ChannelInfo, ChannelUpdateInfo, NodeAnnouncementInfo, NodeInfo}; pub use crate::liquidity::{LSPS1OrderStatus, LSPS2ServiceConfig, OnchainPaymentInfo, PaymentInfo}; From 70014d697e3dbfa7cb7be4adac9c81ba177d6bb6 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 7 Mar 2025 13:46:19 +0100 Subject: [PATCH 06/13] Implement (cached) `Filter` for `ElectrumRuntimeClient` Currently, we won't have a `Runtime` available when initializing `ChainSource::Electrum`. We therefore isolate any runtime-specific behavior into the `ElectrumRuntimeStatus`. Here, we implement `Filter` for `ElectrumRuntimeClient`, but we need to cache the registrations as they might happen prior to `ElectrumRuntimeClient` becoming available. --- src/chain/electrum.rs | 12 +++++++++ src/chain/mod.rs | 58 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index 4e62a75a1..3e1f1465e 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -8,6 +8,7 @@ use crate::error::Error; use crate::logger::{log_error, LdkLogger, Logger}; +use lightning::chain::{Filter, WatchedOutput}; use lightning_transaction_sync::ElectrumSyncClient; use bdk_electrum::BdkElectrumClient; @@ -15,6 +16,8 @@ use bdk_electrum::BdkElectrumClient; use electrum_client::Client as ElectrumClient; use electrum_client::ConfigBuilder as ElectrumConfigBuilder; +use bitcoin::{Script, Txid}; + use std::sync::Arc; const ELECTRUM_CLIENT_NUM_RETRIES: u8 = 3; @@ -58,3 +61,12 @@ impl ElectrumRuntimeClient { Ok(Self { electrum_client, bdk_electrum_client, tx_sync, runtime, logger }) } } + +impl Filter for ElectrumRuntimeClient { + fn register_tx(&self, txid: &Txid, script_pubkey: &Script) { + self.tx_sync.register_tx(txid, script_pubkey) + } + fn register_output(&self, output: WatchedOutput) { + self.tx_sync.register_output(output) + } +} diff --git a/src/chain/mod.rs b/src/chain/mod.rs index ad213405a..5de9a8bc1 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -28,7 +28,7 @@ use crate::types::{Broadcaster, ChainMonitor, ChannelManager, DynStore, Sweeper, use crate::{Error, NodeMetrics}; use lightning::chain::chaininterface::ConfirmationTarget as LdkConfirmationTarget; -use lightning::chain::{Confirm, Filter, Listen}; +use lightning::chain::{Confirm, Filter, Listen, WatchedOutput}; use lightning::util::ser::Writeable; use lightning_transaction_sync::EsploraSyncClient; @@ -42,7 +42,7 @@ use bdk_esplora::EsploraAsyncExt; use esplora_client::AsyncClient as EsploraAsyncClient; -use bitcoin::{FeeRate, Network}; +use bitcoin::{FeeRate, Network, Script, ScriptBuf, Txid}; use std::collections::HashMap; use std::sync::{Arc, Mutex, RwLock}; @@ -111,22 +111,36 @@ impl WalletSyncStatus { pub(crate) enum ElectrumRuntimeStatus { Started(Arc), - Stopped, + Stopped { + pending_registered_txs: Vec<(Txid, ScriptBuf)>, + pending_registered_outputs: Vec, + }, } impl ElectrumRuntimeStatus { pub(crate) fn new() -> Self { - Self::Stopped + let pending_registered_txs = Vec::new(); + let pending_registered_outputs = Vec::new(); + Self::Stopped { pending_registered_txs, pending_registered_outputs } } pub(crate) fn start( &mut self, server_url: String, runtime: Arc, logger: Arc, ) -> Result<(), Error> { match self { - Self::Stopped => { + Self::Stopped { pending_registered_txs, pending_registered_outputs } => { let client = Arc::new(ElectrumRuntimeClient::new(server_url.clone(), runtime, logger)?); + // Apply any pending `Filter` entries + for (txid, script_pubkey) in pending_registered_txs.drain(..) { + client.register_tx(&txid, &script_pubkey); + } + + for output in pending_registered_outputs.drain(..) { + client.register_output(output) + } + *self = Self::Started(client); }, Self::Started(_) => { @@ -140,12 +154,30 @@ impl ElectrumRuntimeStatus { *self = Self::new() } - pub(crate) fn client(&self) -> Option<&Arc> { + pub(crate) fn client(&self) -> Option<&ElectrumRuntimeClient> { match self { - Self::Started(client) => Some(&client), + Self::Started(client) => Some(&*client), Self::Stopped { .. } => None, } } + + fn register_tx(&mut self, txid: &Txid, script_pubkey: &Script) { + match self { + Self::Started(client) => client.register_tx(txid, script_pubkey), + Self::Stopped { pending_registered_txs, .. } => { + pending_registered_txs.push((*txid, script_pubkey.to_owned())) + }, + } + } + + fn register_output(&mut self, output: lightning::chain::WatchedOutput) { + match self { + Self::Started(client) => client.register_output(output), + Self::Stopped { pending_registered_outputs, .. } => { + pending_registered_outputs.push(output) + }, + } + } } pub(crate) enum ChainSource { @@ -278,7 +310,7 @@ impl ChainSource { Self::Electrum { server_url, electrum_runtime_status, logger, .. } => { electrum_runtime_status.write().unwrap().start( server_url.clone(), - runtime, + Arc::clone(&runtime), Arc::clone(&logger), )?; }, @@ -1261,17 +1293,21 @@ impl ChainSource { } impl Filter for ChainSource { - fn register_tx(&self, txid: &bitcoin::Txid, script_pubkey: &bitcoin::Script) { + fn register_tx(&self, txid: &Txid, script_pubkey: &Script) { match self { Self::Esplora { tx_sync, .. } => tx_sync.register_tx(txid, script_pubkey), - Self::Electrum { .. } => todo!(), + Self::Electrum { electrum_runtime_status, .. } => { + electrum_runtime_status.write().unwrap().register_tx(txid, script_pubkey) + }, Self::BitcoindRpc { .. } => (), } } fn register_output(&self, output: lightning::chain::WatchedOutput) { match self { Self::Esplora { tx_sync, .. } => tx_sync.register_output(output), - Self::Electrum { .. } => todo!(), + Self::Electrum { electrum_runtime_status, .. } => { + electrum_runtime_status.write().unwrap().register_output(output) + }, Self::BitcoindRpc { .. } => (), } } From 6ead9be0f97753e1448abcbaef02bcf1737d9038 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 7 Mar 2025 14:44:16 +0100 Subject: [PATCH 07/13] Implement `continuously_sync_wallets` for `ChainSource::Electrum` --- src/chain/mod.rs | 155 ++++++++++++++++++++++++++++------------------- 1 file changed, 94 insertions(+), 61 deletions(-) diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 5de9a8bc1..647a6ab1d 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -13,10 +13,10 @@ use crate::chain::bitcoind_rpc::{ }; use crate::chain::electrum::ElectrumRuntimeClient; use crate::config::{ - Config, ElectrumSyncConfig, EsploraSyncConfig, BDK_CLIENT_CONCURRENCY, BDK_CLIENT_STOP_GAP, - BDK_WALLET_SYNC_TIMEOUT_SECS, FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS, LDK_WALLET_SYNC_TIMEOUT_SECS, - RESOLVED_CHANNEL_MONITOR_ARCHIVAL_INTERVAL, TX_BROADCAST_TIMEOUT_SECS, - WALLET_SYNC_INTERVAL_MINIMUM_SECS, + BackgroundSyncConfig, Config, ElectrumSyncConfig, EsploraSyncConfig, BDK_CLIENT_CONCURRENCY, + BDK_CLIENT_STOP_GAP, BDK_WALLET_SYNC_TIMEOUT_SECS, FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS, + LDK_WALLET_SYNC_TIMEOUT_SECS, RESOLVED_CHANNEL_MONITOR_ARCHIVAL_INTERVAL, + TX_BROADCAST_TIMEOUT_SECS, WALLET_SYNC_INTERVAL_MINIMUM_SECS, }; use crate::fee_estimator::{ apply_post_estimation_adjustments, get_all_conf_targets, get_num_block_defaults_for_target, @@ -346,71 +346,45 @@ impl ChainSource { ) { match self { Self::Esplora { sync_config, logger, .. } => { - // Setup syncing intervals if enabled - if let Some(background_sync_config) = sync_config.background_sync_config { - let onchain_wallet_sync_interval_secs = background_sync_config - .onchain_wallet_sync_interval_secs - .max(WALLET_SYNC_INTERVAL_MINIMUM_SECS); - let mut onchain_wallet_sync_interval = tokio::time::interval( - Duration::from_secs(onchain_wallet_sync_interval_secs), - ); - onchain_wallet_sync_interval - .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - let fee_rate_cache_update_interval_secs = background_sync_config - .fee_rate_cache_update_interval_secs - .max(WALLET_SYNC_INTERVAL_MINIMUM_SECS); - let mut fee_rate_update_interval = tokio::time::interval(Duration::from_secs( - fee_rate_cache_update_interval_secs, - )); - // When starting up, we just blocked on updating, so skip the first tick. - fee_rate_update_interval.reset(); - fee_rate_update_interval - .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - let lightning_wallet_sync_interval_secs = background_sync_config - .lightning_wallet_sync_interval_secs - .max(WALLET_SYNC_INTERVAL_MINIMUM_SECS); - let mut lightning_wallet_sync_interval = tokio::time::interval( - Duration::from_secs(lightning_wallet_sync_interval_secs), + if let Some(background_sync_config) = sync_config.background_sync_config.as_ref() { + self.start_tx_based_sync_loop( + stop_sync_receiver, + channel_manager, + chain_monitor, + output_sweeper, + background_sync_config, + Arc::clone(&logger), + ) + .await + } else { + // Background syncing is disabled + log_info!( + logger, + "Background syncing is disabled. Manual syncing required for onchain wallet, lightning wallet, and fee rate updates.", ); - lightning_wallet_sync_interval - .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - // Start the syncing loop. - loop { - tokio::select! { - _ = stop_sync_receiver.changed() => { - log_trace!( - logger, - "Stopping background syncing on-chain wallet.", - ); - return; - } - _ = onchain_wallet_sync_interval.tick() => { - let _ = self.sync_onchain_wallet().await; - } - _ = fee_rate_update_interval.tick() => { - let _ = self.update_fee_rate_estimates().await; - } - _ = lightning_wallet_sync_interval.tick() => { - let _ = self.sync_lightning_wallet( - Arc::clone(&channel_manager), - Arc::clone(&chain_monitor), - Arc::clone(&output_sweeper), - ).await; - } - } - } + return; + } + }, + Self::Electrum { sync_config, logger, .. } => { + if let Some(background_sync_config) = sync_config.background_sync_config.as_ref() { + self.start_tx_based_sync_loop( + stop_sync_receiver, + channel_manager, + chain_monitor, + output_sweeper, + background_sync_config, + Arc::clone(&logger), + ) + .await } else { // Background syncing is disabled log_info!( - logger, "Background syncing disabled. Manual syncing required for onchain wallet, lightning wallet, and fee rate updates.", + logger, + "Background syncing is disabled. Manual syncing required for onchain wallet, lightning wallet, and fee rate updates.", ); return; } }, - Self::Electrum { .. } => todo!(), Self::BitcoindRpc { bitcoind_rpc_client, header_cache, @@ -561,6 +535,65 @@ impl ChainSource { } } + async fn start_tx_based_sync_loop( + &self, mut stop_sync_receiver: tokio::sync::watch::Receiver<()>, + channel_manager: Arc, chain_monitor: Arc, + output_sweeper: Arc, background_sync_config: &BackgroundSyncConfig, + logger: Arc, + ) { + // Setup syncing intervals + let onchain_wallet_sync_interval_secs = background_sync_config + .onchain_wallet_sync_interval_secs + .max(WALLET_SYNC_INTERVAL_MINIMUM_SECS); + let mut onchain_wallet_sync_interval = + tokio::time::interval(Duration::from_secs(onchain_wallet_sync_interval_secs)); + onchain_wallet_sync_interval + .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + let fee_rate_cache_update_interval_secs = background_sync_config + .fee_rate_cache_update_interval_secs + .max(WALLET_SYNC_INTERVAL_MINIMUM_SECS); + let mut fee_rate_update_interval = + tokio::time::interval(Duration::from_secs(fee_rate_cache_update_interval_secs)); + // When starting up, we just blocked on updating, so skip the first tick. + fee_rate_update_interval.reset(); + fee_rate_update_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + let lightning_wallet_sync_interval_secs = background_sync_config + .lightning_wallet_sync_interval_secs + .max(WALLET_SYNC_INTERVAL_MINIMUM_SECS); + let mut lightning_wallet_sync_interval = + tokio::time::interval(Duration::from_secs(lightning_wallet_sync_interval_secs)); + lightning_wallet_sync_interval + .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + // Start the syncing loop. + loop { + tokio::select! { + _ = stop_sync_receiver.changed() => { + log_trace!( + logger, + "Stopping background syncing on-chain wallet.", + ); + return; + } + _ = onchain_wallet_sync_interval.tick() => { + let _ = self.sync_onchain_wallet().await; + } + _ = fee_rate_update_interval.tick() => { + let _ = self.update_fee_rate_estimates().await; + } + _ = lightning_wallet_sync_interval.tick() => { + let _ = self.sync_lightning_wallet( + Arc::clone(&channel_manager), + Arc::clone(&chain_monitor), + Arc::clone(&output_sweeper), + ).await; + } + } + } + } + // Synchronize the onchain wallet via transaction-based protocols (i.e., Esplora, Electrum, // etc.) pub(crate) async fn sync_onchain_wallet(&self) -> Result<(), Error> { From 98150cdeec01bbd8359bc446a37545998fbdbe75 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 7 Mar 2025 15:37:25 +0100 Subject: [PATCH 08/13] Implement `process_broadcast_queue` for `ChainSource::Electrum` --- src/chain/electrum.rs | 50 +++++++++++++++++++++++++++++++++++++++++-- src/chain/mod.rs | 25 +++++++++++++++++++--- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index 3e1f1465e..ea98fd91c 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -5,20 +5,24 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. +use crate::config::TX_BROADCAST_TIMEOUT_SECS; use crate::error::Error; -use crate::logger::{log_error, LdkLogger, Logger}; +use crate::logger::{log_bytes, log_error, log_trace, LdkLogger, Logger}; use lightning::chain::{Filter, WatchedOutput}; +use lightning::util::ser::Writeable; use lightning_transaction_sync::ElectrumSyncClient; use bdk_electrum::BdkElectrumClient; use electrum_client::Client as ElectrumClient; use electrum_client::ConfigBuilder as ElectrumConfigBuilder; +use electrum_client::ElectrumApi; -use bitcoin::{Script, Txid}; +use bitcoin::{Script, Transaction, Txid}; use std::sync::Arc; +use std::time::Duration; const ELECTRUM_CLIENT_NUM_RETRIES: u8 = 3; const ELECTRUM_CLIENT_TIMEOUT_SECS: u8 = 20; @@ -60,6 +64,48 @@ impl ElectrumRuntimeClient { ); Ok(Self { electrum_client, bdk_electrum_client, tx_sync, runtime, logger }) } + + pub(crate) async fn broadcast(&self, tx: Transaction) { + let electrum_client = Arc::clone(&self.electrum_client); + + let txid = tx.compute_txid(); + let tx_bytes = tx.encode(); + + let spawn_fut = + self.runtime.spawn_blocking(move || electrum_client.transaction_broadcast(&tx)); + + let timeout_fut = + tokio::time::timeout(Duration::from_secs(TX_BROADCAST_TIMEOUT_SECS), spawn_fut); + + match timeout_fut.await { + Ok(res) => match res { + Ok(_) => { + log_trace!(self.logger, "Successfully broadcast transaction {}", txid); + }, + Err(e) => { + log_error!(self.logger, "Failed to broadcast transaction {}: {}", txid, e); + log_trace!( + self.logger, + "Failed broadcast transaction bytes: {}", + log_bytes!(tx_bytes) + ); + }, + }, + Err(e) => { + log_error!( + self.logger, + "Failed to broadcast transaction due to timeout {}: {}", + txid, + e + ); + log_trace!( + self.logger, + "Failed broadcast transaction bytes: {}", + log_bytes!(tx_bytes) + ); + }, + } + } } impl Filter for ElectrumRuntimeClient { diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 647a6ab1d..f6a30c4d0 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -154,9 +154,9 @@ impl ElectrumRuntimeStatus { *self = Self::new() } - pub(crate) fn client(&self) -> Option<&ElectrumRuntimeClient> { + pub(crate) fn client(&self) -> Option> { match self { - Self::Started(client) => Some(&*client), + Self::Started(client) => Some(Arc::clone(&client)), Self::Stopped { .. } => None, } } @@ -1266,7 +1266,26 @@ impl ChainSource { } } }, - Self::Electrum { .. } => todo!(), + Self::Electrum { electrum_runtime_status, tx_broadcaster, .. } => { + let electrum_client: Arc = if let Some(client) = + electrum_runtime_status.read().unwrap().client().as_ref() + { + Arc::clone(client) + } else { + debug_assert!( + false, + "We should have started the chain source before broadcasting" + ); + return; + }; + + let mut receiver = tx_broadcaster.get_broadcast_queue().await; + while let Some(next_package) = receiver.recv().await { + for tx in next_package { + electrum_client.broadcast(tx).await; + } + } + }, Self::BitcoindRpc { bitcoind_rpc_client, tx_broadcaster, logger, .. } => { // While it's a bit unclear when we'd be able to lean on Bitcoin Core >v28 // features, we should eventually switch to use `submitpackage` via the From af64dd9ff383552b81b10e19366c609c9a4c25f3 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 17 Mar 2025 11:05:48 +0100 Subject: [PATCH 09/13] Implement `update_fee_rate_estimates` for `ChainSource::Electrum` --- src/chain/electrum.rs | 100 +++++++++++++++++++++++++++++++++++++++--- src/chain/mod.rs | 60 ++++++++++++++++++++++--- 2 files changed, 150 insertions(+), 10 deletions(-) diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index ea98fd91c..838ff78f7 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -5,8 +5,12 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. -use crate::config::TX_BROADCAST_TIMEOUT_SECS; +use crate::config::{Config, FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS, TX_BROADCAST_TIMEOUT_SECS}; use crate::error::Error; +use crate::fee_estimator::{ + apply_post_estimation_adjustments, get_all_conf_targets, get_num_block_defaults_for_target, + ConfirmationTarget, +}; use crate::logger::{log_bytes, log_error, log_trace, LdkLogger, Logger}; use lightning::chain::{Filter, WatchedOutput}; @@ -17,10 +21,11 @@ use bdk_electrum::BdkElectrumClient; use electrum_client::Client as ElectrumClient; use electrum_client::ConfigBuilder as ElectrumConfigBuilder; -use electrum_client::ElectrumApi; +use electrum_client::{Batch, ElectrumApi}; -use bitcoin::{Script, Transaction, Txid}; +use bitcoin::{FeeRate, Network, Script, Transaction, Txid}; +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -32,12 +37,14 @@ pub(crate) struct ElectrumRuntimeClient { bdk_electrum_client: Arc>, tx_sync: Arc>>, runtime: Arc, + config: Arc, logger: Arc, } impl ElectrumRuntimeClient { pub(crate) fn new( - server_url: String, runtime: Arc, logger: Arc, + server_url: String, runtime: Arc, config: Arc, + logger: Arc, ) -> Result { let electrum_config = ElectrumConfigBuilder::new() .retry(ELECTRUM_CLIENT_NUM_RETRIES) @@ -62,7 +69,7 @@ impl ElectrumRuntimeClient { Error::ConnectionFailed })?, ); - Ok(Self { electrum_client, bdk_electrum_client, tx_sync, runtime, logger }) + Ok(Self { electrum_client, bdk_electrum_client, tx_sync, runtime, config, logger }) } pub(crate) async fn broadcast(&self, tx: Transaction) { @@ -106,6 +113,89 @@ impl ElectrumRuntimeClient { }, } } + + pub(crate) async fn get_fee_rate_cache_update( + &self, + ) -> Result, Error> { + let electrum_client = Arc::clone(&self.electrum_client); + + let mut batch = Batch::default(); + let confirmation_targets = get_all_conf_targets(); + for target in confirmation_targets { + let num_blocks = get_num_block_defaults_for_target(target); + batch.estimate_fee(num_blocks); + } + + let spawn_fut = self.runtime.spawn_blocking(move || electrum_client.batch_call(&batch)); + + let timeout_fut = tokio::time::timeout( + Duration::from_secs(FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS), + spawn_fut, + ); + + let raw_estimates_btc_kvb = timeout_fut + .await + .map_err(|e| { + log_error!(self.logger, "Updating fee rate estimates timed out: {}", e); + Error::FeerateEstimationUpdateTimeout + })? + .map_err(|e| { + log_error!(self.logger, "Failed to retrieve fee rate estimates: {}", e); + Error::FeerateEstimationUpdateFailed + })? + .map_err(|e| { + log_error!(self.logger, "Failed to retrieve fee rate estimates: {}", e); + Error::FeerateEstimationUpdateFailed + })?; + + if raw_estimates_btc_kvb.len() != confirmation_targets.len() + && self.config.network == Network::Bitcoin + { + // Ensure we fail if we didn't receive all estimates. + debug_assert!(false, + "Electrum server didn't return all expected results. This is disallowed on Mainnet." + ); + log_error!(self.logger, + "Failed to retrieve fee rate estimates: Electrum server didn't return all expected results. This is disallowed on Mainnet." + ); + return Err(Error::FeerateEstimationUpdateFailed); + } + + let mut new_fee_rate_cache = HashMap::with_capacity(10); + for (target, raw_fee_rate_btc_per_kvb) in + confirmation_targets.into_iter().zip(raw_estimates_btc_kvb.into_iter()) + { + // Parse the retrieved serde_json::Value and fall back to 1 sat/vb (10^3 / 10^8 = 10^-5 + // = 0.00001 btc/kvb) if we fail or it yields less than that. This is mostly necessary + // to continue on `signet`/`regtest` where we might not get estimates (or bogus + // values). + let fee_rate_btc_per_kvb = raw_fee_rate_btc_per_kvb + .as_f64() + .map_or(0.00001, |converted| converted.max(0.00001)); + + // Electrum, just like Bitcoin Core, gives us a feerate in BTC/KvB. + // Thus, we multiply by 25_000_000 (10^8 / 4) to get satoshis/kwu. + let fee_rate = { + let fee_rate_sat_per_kwu = (fee_rate_btc_per_kvb * 25_000_000.0).round() as u64; + FeeRate::from_sat_per_kwu(fee_rate_sat_per_kwu) + }; + + // LDK 0.0.118 introduced changes to the `ConfirmationTarget` semantics that + // require some post-estimation adjustments to the fee rates, which we do here. + let adjusted_fee_rate = apply_post_estimation_adjustments(target, fee_rate); + + new_fee_rate_cache.insert(target, adjusted_fee_rate); + + log_trace!( + self.logger, + "Fee rate estimation updated for {:?}: {} sats/kwu", + target, + adjusted_fee_rate.to_sat_per_kwu(), + ); + } + + Ok(new_fee_rate_cache) + } } impl Filter for ElectrumRuntimeClient { diff --git a/src/chain/mod.rs b/src/chain/mod.rs index f6a30c4d0..75f0adaeb 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -125,12 +125,17 @@ impl ElectrumRuntimeStatus { } pub(crate) fn start( - &mut self, server_url: String, runtime: Arc, logger: Arc, + &mut self, server_url: String, runtime: Arc, config: Arc, + logger: Arc, ) -> Result<(), Error> { match self { Self::Stopped { pending_registered_txs, pending_registered_outputs } => { - let client = - Arc::new(ElectrumRuntimeClient::new(server_url.clone(), runtime, logger)?); + let client = Arc::new(ElectrumRuntimeClient::new( + server_url.clone(), + runtime, + config, + logger, + )?); // Apply any pending `Filter` entries for (txid, script_pubkey) in pending_registered_txs.drain(..) { @@ -307,10 +312,11 @@ impl ChainSource { pub(crate) fn start(&self, runtime: Arc) -> Result<(), Error> { match self { - Self::Electrum { server_url, electrum_runtime_status, logger, .. } => { + Self::Electrum { server_url, electrum_runtime_status, config, logger, .. } => { electrum_runtime_status.write().unwrap().start( server_url.clone(), Arc::clone(&runtime), + Arc::clone(&config), Arc::clone(&logger), )?; }, @@ -1055,7 +1061,51 @@ impl ChainSource { Ok(()) }, - Self::Electrum { .. } => todo!(), + Self::Electrum { + electrum_runtime_status, + fee_estimator, + kv_store, + logger, + node_metrics, + .. + } => { + let electrum_client: Arc = if let Some(client) = + electrum_runtime_status.read().unwrap().client().as_ref() + { + Arc::clone(client) + } else { + debug_assert!( + false, + "We should have started the chain source before updating fees" + ); + return Err(Error::FeerateEstimationUpdateFailed); + }; + + let now = Instant::now(); + + let new_fee_rate_cache = electrum_client.get_fee_rate_cache_update().await?; + fee_estimator.set_fee_rate_cache(new_fee_rate_cache); + + log_info!( + logger, + "Fee rate cache update finished in {}ms.", + now.elapsed().as_millis() + ); + + let unix_time_secs_opt = + SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); + { + let mut locked_node_metrics = node_metrics.write().unwrap(); + locked_node_metrics.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt; + write_node_metrics( + &*locked_node_metrics, + Arc::clone(&kv_store), + Arc::clone(&logger), + )?; + } + + Ok(()) + }, Self::BitcoindRpc { bitcoind_rpc_client, fee_estimator, From 6ac8b914f05a3ec72cf999d32ed23df24e9aacf7 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 17 Mar 2025 11:36:58 +0100 Subject: [PATCH 10/13] Implement `sync_lightning_wallet` for `ChainSource::Electrum` --- src/chain/electrum.rs | 45 ++++++++++++++++++++++++--- src/chain/mod.rs | 72 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index 838ff78f7..5907357d1 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -5,15 +5,18 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. -use crate::config::{Config, FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS, TX_BROADCAST_TIMEOUT_SECS}; +use crate::config::{ + Config, FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS, LDK_WALLET_SYNC_TIMEOUT_SECS, + TX_BROADCAST_TIMEOUT_SECS, +}; use crate::error::Error; use crate::fee_estimator::{ apply_post_estimation_adjustments, get_all_conf_targets, get_num_block_defaults_for_target, ConfirmationTarget, }; -use crate::logger::{log_bytes, log_error, log_trace, LdkLogger, Logger}; +use crate::logger::{log_bytes, log_error, log_info, log_trace, LdkLogger, Logger}; -use lightning::chain::{Filter, WatchedOutput}; +use lightning::chain::{Confirm, Filter, WatchedOutput}; use lightning::util::ser::Writeable; use lightning_transaction_sync::ElectrumSyncClient; @@ -27,7 +30,7 @@ use bitcoin::{FeeRate, Network, Script, Transaction, Txid}; use std::collections::HashMap; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; const ELECTRUM_CLIENT_NUM_RETRIES: u8 = 3; const ELECTRUM_CLIENT_TIMEOUT_SECS: u8 = 20; @@ -72,6 +75,40 @@ impl ElectrumRuntimeClient { Ok(Self { electrum_client, bdk_electrum_client, tx_sync, runtime, config, logger }) } + pub(crate) async fn sync_confirmables( + &self, confirmables: Vec>, + ) -> Result<(), Error> { + let now = Instant::now(); + + let tx_sync = Arc::clone(&self.tx_sync); + let spawn_fut = self.runtime.spawn_blocking(move || tx_sync.sync(confirmables)); + let timeout_fut = + tokio::time::timeout(Duration::from_secs(LDK_WALLET_SYNC_TIMEOUT_SECS), spawn_fut); + + let res = timeout_fut + .await + .map_err(|e| { + log_error!(self.logger, "Sync of Lightning wallet timed out: {}", e); + Error::TxSyncTimeout + })? + .map_err(|e| { + log_error!(self.logger, "Sync of Lightning wallet failed: {}", e); + Error::TxSyncFailed + })? + .map_err(|e| { + log_error!(self.logger, "Sync of Lightning wallet failed: {}", e); + Error::TxSyncFailed + })?; + + log_info!( + self.logger, + "Sync of Lightning wallet finished in {}ms.", + now.elapsed().as_millis() + ); + + Ok(res) + } + pub(crate) async fn broadcast(&self, tx: Transaction) { let electrum_client = Arc::clone(&self.electrum_client); diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 75f0adaeb..57f4d9f65 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -817,7 +817,77 @@ impl ChainSource { res }, - Self::Electrum { .. } => todo!(), + Self::Electrum { + electrum_runtime_status, + lightning_wallet_sync_status, + kv_store, + logger, + node_metrics, + .. + } => { + let electrum_client: Arc = if let Some(client) = + electrum_runtime_status.read().unwrap().client().as_ref() + { + Arc::clone(client) + } else { + debug_assert!( + false, + "We should have started the chain source before syncing the lightning wallet" + ); + return Err(Error::TxSyncFailed); + }; + + let sync_cman = Arc::clone(&channel_manager); + let sync_cmon = Arc::clone(&chain_monitor); + let sync_sweeper = Arc::clone(&output_sweeper); + let confirmables = vec![ + sync_cman as Arc, + sync_cmon as Arc, + sync_sweeper as Arc, + ]; + + let receiver_res = { + let mut status_lock = lightning_wallet_sync_status.lock().unwrap(); + status_lock.register_or_subscribe_pending_sync() + }; + if let Some(mut sync_receiver) = receiver_res { + log_info!(logger, "Sync in progress, skipping."); + return sync_receiver.recv().await.map_err(|e| { + debug_assert!(false, "Failed to receive wallet sync result: {:?}", e); + log_error!(logger, "Failed to receive wallet sync result: {:?}", e); + Error::TxSyncFailed + })?; + } + + let res = electrum_client.sync_confirmables(confirmables).await; + + if let Ok(_) = res { + let unix_time_secs_opt = + SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); + { + let mut locked_node_metrics = node_metrics.write().unwrap(); + locked_node_metrics.latest_lightning_wallet_sync_timestamp = + unix_time_secs_opt; + write_node_metrics( + &*locked_node_metrics, + Arc::clone(&kv_store), + Arc::clone(&logger), + )?; + } + + periodically_archive_fully_resolved_monitors( + Arc::clone(&channel_manager), + Arc::clone(&chain_monitor), + Arc::clone(&kv_store), + Arc::clone(&logger), + Arc::clone(&node_metrics), + )?; + } + + lightning_wallet_sync_status.lock().unwrap().propagate_result_to_subscribers(res); + + res + }, Self::BitcoindRpc { .. } => { // In BitcoindRpc mode we sync lightning and onchain wallet in one go by via // `ChainPoller`. So nothing to do here. From 2bd559bd83519e26bedd55c0df07f47fd5ae7e92 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 17 Mar 2025 13:58:53 +0100 Subject: [PATCH 11/13] Implement `sync_onchain_wallet` for `ChainSource::Electrum` --- src/chain/electrum.rs | 74 +++++++++++++++++++++++++++++++++- src/chain/mod.rs | 94 ++++++++++++++++++++++++++++++++++++++++++- src/wallet/mod.rs | 4 ++ 3 files changed, 169 insertions(+), 3 deletions(-) diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index 5907357d1..6e62d9c08 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -6,8 +6,8 @@ // accordance with one or both of these licenses. use crate::config::{ - Config, FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS, LDK_WALLET_SYNC_TIMEOUT_SECS, - TX_BROADCAST_TIMEOUT_SECS, + Config, BDK_CLIENT_STOP_GAP, BDK_WALLET_SYNC_TIMEOUT_SECS, FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS, + LDK_WALLET_SYNC_TIMEOUT_SECS, TX_BROADCAST_TIMEOUT_SECS, }; use crate::error::Error; use crate::fee_estimator::{ @@ -20,6 +20,12 @@ use lightning::chain::{Confirm, Filter, WatchedOutput}; use lightning::util::ser::Writeable; use lightning_transaction_sync::ElectrumSyncClient; +use bdk_chain::bdk_core::spk_client::FullScanRequest as BdkFullScanRequest; +use bdk_chain::bdk_core::spk_client::FullScanResponse as BdkFullScanResponse; +use bdk_chain::bdk_core::spk_client::SyncRequest as BdkSyncRequest; +use bdk_chain::bdk_core::spk_client::SyncResponse as BdkSyncResponse; +use bdk_wallet::KeychainKind as BdkKeyChainKind; + use bdk_electrum::BdkElectrumClient; use electrum_client::Client as ElectrumClient; @@ -32,6 +38,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; +const BDK_ELECTRUM_CLIENT_BATCH_SIZE: usize = 5; const ELECTRUM_CLIENT_NUM_RETRIES: u8 = 3; const ELECTRUM_CLIENT_TIMEOUT_SECS: u8 = 20; @@ -109,6 +116,69 @@ impl ElectrumRuntimeClient { Ok(res) } + pub(crate) async fn get_full_scan_wallet_update( + &self, request: BdkFullScanRequest, + cached_txs: impl IntoIterator>>, + ) -> Result, Error> { + let bdk_electrum_client = Arc::clone(&self.bdk_electrum_client); + bdk_electrum_client.populate_tx_cache(cached_txs); + + let spawn_fut = self.runtime.spawn_blocking(move || { + bdk_electrum_client.full_scan( + request, + BDK_CLIENT_STOP_GAP, + BDK_ELECTRUM_CLIENT_BATCH_SIZE, + true, + ) + }); + let wallet_sync_timeout_fut = + tokio::time::timeout(Duration::from_secs(BDK_WALLET_SYNC_TIMEOUT_SECS), spawn_fut); + + wallet_sync_timeout_fut + .await + .map_err(|e| { + log_error!(self.logger, "Sync of on-chain wallet timed out: {}", e); + Error::WalletOperationTimeout + })? + .map_err(|e| { + log_error!(self.logger, "Sync of on-chain wallet failed: {}", e); + Error::WalletOperationFailed + })? + .map_err(|e| { + log_error!(self.logger, "Sync of on-chain wallet failed: {}", e); + Error::WalletOperationFailed + }) + } + + pub(crate) async fn get_incremental_sync_wallet_update( + &self, request: BdkSyncRequest<(BdkKeyChainKind, u32)>, + cached_txs: impl IntoIterator>>, + ) -> Result { + let bdk_electrum_client = Arc::clone(&self.bdk_electrum_client); + bdk_electrum_client.populate_tx_cache(cached_txs); + + let spawn_fut = self.runtime.spawn_blocking(move || { + bdk_electrum_client.sync(request, BDK_ELECTRUM_CLIENT_BATCH_SIZE, true) + }); + let wallet_sync_timeout_fut = + tokio::time::timeout(Duration::from_secs(BDK_WALLET_SYNC_TIMEOUT_SECS), spawn_fut); + + wallet_sync_timeout_fut + .await + .map_err(|e| { + log_error!(self.logger, "Incremental sync of on-chain wallet timed out: {}", e); + Error::WalletOperationTimeout + })? + .map_err(|e| { + log_error!(self.logger, "Incremental sync of on-chain wallet failed: {}", e); + Error::WalletOperationFailed + })? + .map_err(|e| { + log_error!(self.logger, "Incremental sync of on-chain wallet failed: {}", e); + Error::WalletOperationFailed + }) + } + pub(crate) async fn broadcast(&self, tx: Transaction) { let electrum_client = Arc::clone(&self.electrum_client); diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 57f4d9f65..62627797e 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -39,6 +39,7 @@ use lightning_block_sync::poll::{ChainPoller, ChainTip, ValidatedBlockHeader}; use lightning_block_sync::SpvClient; use bdk_esplora::EsploraAsyncExt; +use bdk_wallet::Update as BdkUpdate; use esplora_client::AsyncClient as EsploraAsyncClient; @@ -717,7 +718,98 @@ impl ChainSource { res }, - Self::Electrum { .. } => todo!(), + Self::Electrum { + electrum_runtime_status, + onchain_wallet, + onchain_wallet_sync_status, + kv_store, + logger, + node_metrics, + .. + } => { + let electrum_client: Arc = if let Some(client) = + electrum_runtime_status.read().unwrap().client().as_ref() + { + Arc::clone(client) + } else { + debug_assert!( + false, + "We should have started the chain source before syncing the onchain wallet" + ); + return Err(Error::FeerateEstimationUpdateFailed); + }; + let receiver_res = { + let mut status_lock = onchain_wallet_sync_status.lock().unwrap(); + status_lock.register_or_subscribe_pending_sync() + }; + if let Some(mut sync_receiver) = receiver_res { + log_info!(logger, "Sync in progress, skipping."); + return sync_receiver.recv().await.map_err(|e| { + debug_assert!(false, "Failed to receive wallet sync result: {:?}", e); + log_error!(logger, "Failed to receive wallet sync result: {:?}", e); + Error::WalletOperationFailed + })?; + } + + // If this is our first sync, do a full scan with the configured gap limit. + // Otherwise just do an incremental sync. + let incremental_sync = + node_metrics.read().unwrap().latest_onchain_wallet_sync_timestamp.is_some(); + + let apply_wallet_update = + |update_res: Result, now: Instant| match update_res { + Ok(update) => match onchain_wallet.apply_update(update) { + Ok(()) => { + log_info!( + logger, + "{} of on-chain wallet finished in {}ms.", + if incremental_sync { "Incremental sync" } else { "Sync" }, + now.elapsed().as_millis() + ); + let unix_time_secs_opt = SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .map(|d| d.as_secs()); + { + let mut locked_node_metrics = node_metrics.write().unwrap(); + locked_node_metrics.latest_onchain_wallet_sync_timestamp = + unix_time_secs_opt; + write_node_metrics( + &*locked_node_metrics, + Arc::clone(&kv_store), + Arc::clone(&logger), + )?; + } + Ok(()) + }, + Err(e) => Err(e), + }, + Err(e) => Err(e), + }; + + let cached_txs = onchain_wallet.get_cached_txs(); + + let res = if incremental_sync { + let incremental_sync_request = onchain_wallet.get_incremental_sync_request(); + let incremental_sync_fut = electrum_client + .get_incremental_sync_wallet_update(incremental_sync_request, cached_txs); + + let now = Instant::now(); + let update_res = incremental_sync_fut.await.map(|u| u.into()); + apply_wallet_update(update_res, now) + } else { + let full_scan_request = onchain_wallet.get_full_scan_request(); + let full_scan_fut = + electrum_client.get_full_scan_wallet_update(full_scan_request, cached_txs); + let now = Instant::now(); + let update_res = full_scan_fut.await.map(|u| u.into()); + apply_wallet_update(update_res, now) + }; + + onchain_wallet_sync_status.lock().unwrap().propagate_result_to_subscribers(res); + + res + }, Self::BitcoindRpc { .. } => { // In BitcoindRpc mode we sync lightning and onchain wallet in one go by via // `ChainPoller`. So nothing to do here. diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 58a28308d..53de4b8d5 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -98,6 +98,10 @@ where self.inner.lock().unwrap().start_sync_with_revealed_spks().build() } + pub(crate) fn get_cached_txs(&self) -> Vec> { + self.inner.lock().unwrap().tx_graph().full_txs().map(|tx_node| tx_node.tx).collect() + } + pub(crate) fn current_best_block(&self) -> BestBlock { let checkpoint = self.inner.lock().unwrap().latest_checkpoint(); BestBlock { block_hash: checkpoint.hash(), height: checkpoint.height() } From 95e75d497003b634a9da063362219b897d47d701 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 17 Mar 2025 14:12:07 +0100 Subject: [PATCH 12/13] Add `full_cycle` test using the new `Electrum` chain source .. as we do with `BitcoindRpc`, we now test `Electrum` support by running the `full_cycle` test in CI. --- tests/common/mod.rs | 8 +++++++- tests/integration_tests_rust.rs | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index ac8d1ce90..3258df791 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -12,7 +12,7 @@ pub(crate) mod logging; use logging::TestLogWriter; -use ldk_node::config::{Config, EsploraSyncConfig}; +use ldk_node::config::{Config, ElectrumSyncConfig, EsploraSyncConfig}; use ldk_node::io::sqlite_store::SqliteStore; use ldk_node::payment::{PaymentDirection, PaymentKind, PaymentStatus}; use ldk_node::{ @@ -255,6 +255,7 @@ type TestNode = Node; #[derive(Clone)] pub(crate) enum TestChainSource<'a> { Esplora(&'a ElectrsD), + Electrum(&'a ElectrsD), BitcoindRpc(&'a BitcoinD), } @@ -311,6 +312,11 @@ pub(crate) fn setup_node( let sync_config = EsploraSyncConfig { background_sync_config: None }; builder.set_chain_source_esplora(esplora_url.clone(), Some(sync_config)); }, + TestChainSource::Electrum(electrsd) => { + let electrum_url = format!("tcp://{}", electrsd.electrum_url); + let sync_config = ElectrumSyncConfig { background_sync_config: None }; + builder.set_chain_source_electrum(electrum_url.clone(), Some(sync_config)); + }, TestChainSource::BitcoindRpc(bitcoind) => { let rpc_host = bitcoind.params.rpc_socket.ip().to_string(); let rpc_port = bitcoind.params.rpc_socket.port(); diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index 9bbd56689..f2dfa4b5e 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -45,6 +45,14 @@ fn channel_full_cycle() { do_channel_full_cycle(node_a, node_b, &bitcoind.client, &electrsd.client, false, true, false); } +#[test] +fn channel_full_cycle_electrum() { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + let chain_source = TestChainSource::Electrum(&electrsd); + let (node_a, node_b) = setup_two_nodes(&chain_source, false, true, false); + do_channel_full_cycle(node_a, node_b, &bitcoind.client, &electrsd.client, false, true, false); +} + #[test] fn channel_full_cycle_bitcoind() { let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); From 5cdd2e3a6bd469824db8a206183a8e8772923bc1 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 17 Mar 2025 15:13:00 +0100 Subject: [PATCH 13/13] Update README to signal Electrum support --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f8a3664d4..ed35d8640 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ fn main() { LDK Node currently comes with a decidedly opinionated set of design choices: - On-chain data is handled by the integrated [BDK][bdk] wallet. -- Chain data may currently be sourced from the Bitcoin Core RPC interface or an [Esplora][esplora] server, while support for Electrum will follow soon. +- Chain data may currently be sourced from the Bitcoin Core RPC interface, or from an [Electrum][electrum] or [Esplora][esplora] server. - Wallet and channel state may be persisted to an [SQLite][sqlite] database, to file system, or to a custom back-end to be implemented by the user. - Gossip data may be sourced via Lightning's peer-to-peer network or the [Rapid Gossip Sync](https://docs.rs/lightning-rapid-gossip-sync/*/lightning_rapid_gossip_sync/) protocol. - Entropy for the Lightning and on-chain wallets may be sourced from raw bytes or a [BIP39](https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki) mnemonic. In addition, LDK Node offers the means to generate and persist the entropy bytes to disk. @@ -72,6 +72,7 @@ The Minimum Supported Rust Version (MSRV) is currently 1.75.0. [rust_crate]: https://crates.io/ [ldk]: https://lightningdevkit.org/ [bdk]: https://bitcoindevkit.org/ +[electrum]: https://github.com/spesmilo/electrum-protocol [esplora]: https://github.com/Blockstream/esplora [sqlite]: https://sqlite.org/ [rust]: https://www.rust-lang.org/