feat(fortuna): add option to disable adjustment threads (#2816)

Author: tejasbadadare

apps/fortuna/README.md

@@ -56,48 +56,78 @@ Fortuna supports running multiple replica instances for high availability and re
 - Each replica primarily handles requests assigned to its ID
 - After a configurable delay, replicas will process requests from other replicas as backup (failover)
 
+### Fee Management with Multiple Instances
+
+When running multiple Fortuna instances with different keeper wallets but a single provider, only one instance should handle fee management. This instance needs to run using the same private key as the fee manager, because only the registerd fee manager wallet can adjust fees and withdraw funds.
+
 ### Example Configurations
 
-**Two Replica Setup (Blue/Green):**
+**Two Replica Setup with Fee Management:**
 ```yaml
-# Replica 0 (Blue) - handles even sequence numbers (0, 2, 4, ...)
+# Replica 0 (fee manager wallet) - handles even sequence numbers + fee management
 keeper:
+  private_key:
+    value: 0x<fee_manager_private_key>
   replica_config:
     replica_id: 0
     total_replicas: 2
     backup_delay_seconds: 30
+  run_config:
+    disable_fee_adjustment: false  # Enable fee management (default)
+    disable_fee_withdrawal: false
 
-# Replica 1 (Green) - handles odd sequence numbers (1, 3, 5, ...)
+# Replica 1 (non-fee-manager wallet) - handles odd sequence numbers only
 keeper:
+  private_key:
+    value: 0x<other_keeper_private_key>
   replica_config:
     replica_id: 1
     total_replicas: 2
     backup_delay_seconds: 30
+  run_config:
+    disable_fee_adjustment: true   # Disable fee management
+    disable_fee_withdrawal: true
 ```
 
 **Three Replica Setup:**
 ```yaml
-# Replica 0 - handles sequence numbers 0, 3, 6, 9, ...
+# Replica 0 (fee manager wallet) - handles sequence numbers 0, 3, 6, 9, ... + fee management
 keeper:
   replica_config:
     replica_id: 0
     total_replicas: 3
     backup_delay_seconds: 30
+  run_config:
+    disable_fee_adjustment: false
+    disable_fee_withdrawal: false
+
+# Replicas 1 & 2 (non-fee-manager wallets) - request processing only
+keeper:
+  replica_config:
+    replica_id: 1  # or 2
+    total_replicas: 3
+    backup_delay_seconds: 30
+  run_config:
+    disable_fee_adjustment: true
+    disable_fee_withdrawal: true
 ```
 
 ### Deployment Considerations
 
 1. **Separate Wallets**: Each replica MUST use a different private key to avoid nonce conflicts
-2. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 30-60 seconds)
-3. **Monitoring**: Monitor each replica's processing metrics to ensure proper load distribution
-4. **Gas Management**: Each replica needs sufficient ETH balance for gas fees
+2. **Fee Manager Assignment**: Set the provider's `fee_manager` address to match the primary instance's keeper wallet
+3. **Thread Configuration**: Only enable fee management threads on the instance using the fee manager wallet
+4. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 30-60 seconds)
+5. **Monitoring**: Monitor each replica's processing metrics to ensure proper load distribution
+6. **Gas Management**: Each replica needs sufficient ETH balance for gas fees
 
 ### Failover Behavior
 
 - Primary replica processes requests immediately
 - Backup replicas wait for `backup_delay_seconds` before checking if request is still unfulfilled
 - If request is already fulfilled during the delay, backup replica skips processing
 - This prevents duplicate transactions and wasted gas while ensuring reliability
+- Fee management operations (adjustment/withdrawal) only occur on an instance where the keeper wallet is the fee manager wallet.
 
 ## Local Development
 

apps/fortuna/config.sample.yaml

@@ -87,6 +87,13 @@ keeper:
     # For production, you can store the private key in a file.
     # file: keeper-key.txt
 
+  # Runtime configuration for the keeper service
+  # Optional: Configure which keeper threads to disable. If running multiple replicas,
+  # only a single replica should have the fee adjustment and withdrawal threads enabled.
+  # run_config:
+  #   disable_fee_adjustment: false    # Set to true to disable automatic fee adjustment
+  #   disable_fee_withdrawal: false    # Set to true to disable automatic fee withdrawal
+
   # Multi-replica configuration
   # Optional: Multi-replica configuration for high availability and load distribution
   # Uncomment and configure for production deployments with multiple Fortuna instances

apps/fortuna/src/command/run.rs

@@ -3,7 +3,10 @@ use {
         api::{self, ApiBlockChainState, BlockchainState, ChainId},
         chain::ethereum::InstrumentedPythContract,
         command::register_provider::CommitmentMetadata,
-        config::{Commitment, Config, EthereumConfig, ProviderConfig, ReplicaConfig, RunOptions},
+        config::{
+            Commitment, Config, EthereumConfig, ProviderConfig, ReplicaConfig, RunConfig,
+            RunOptions,
+        },
         eth_utils::traced_client::RpcMetrics,
         history::History,
         keeper::{self, keeper_metrics::KeeperMetrics},
@@ -101,6 +104,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> {
     }
 
     let keeper_replica_config = config.keeper.replica_config.clone();
+    let keeper_run_config = config.keeper.run_config.clone();
 
     let chains: Arc<RwLock<HashMap<ChainId, ApiBlockChainState>>> = Arc::new(RwLock::new(
         config
@@ -115,6 +119,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> {
         let keeper_metrics = keeper_metrics.clone();
         let keeper_private_key_option = keeper_private_key_option.clone();
         let keeper_replica_config = keeper_replica_config.clone();
+        let keeper_run_config = keeper_run_config.clone();
         let chains = chains.clone();
         let secret_copy = secret.clone();
         let rpc_metrics = rpc_metrics.clone();
@@ -129,6 +134,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> {
                     keeper_metrics.clone(),
                     keeper_private_key_option.clone(),
                     keeper_replica_config.clone(),
+                    keeper_run_config.clone(),
                     chains.clone(),
                     &secret_copy,
                     history.clone(),
@@ -180,6 +186,7 @@ async fn setup_chain_and_run_keeper(
     keeper_metrics: Arc<KeeperMetrics>,
     keeper_private_key_option: Option<String>,
     keeper_replica_config: Option<ReplicaConfig>,
+    keeper_run_config: RunConfig,
     chains: Arc<RwLock<HashMap<ChainId, ApiBlockChainState>>>,
     secret_copy: &str,
     history: Arc<History>,
@@ -203,6 +210,7 @@ async fn setup_chain_and_run_keeper(
         keeper::run_keeper_threads(
             keeper_private_key,
             keeper_replica_config,
+            keeper_run_config,
             chain_config,
             state,
             keeper_metrics.clone(),

apps/fortuna/src/config.rs

@@ -350,6 +350,17 @@ fn default_chain_sample_interval() -> u64 {
     1
 }
 
+#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
+pub struct RunConfig {
+    /// Disable automatic fee adjustment threads
+    #[serde(default)]
+    pub disable_fee_adjustment: bool,
+
+    /// Disable automatic fee withdrawal threads
+    #[serde(default)]
+    pub disable_fee_withdrawal: bool,
+}
+
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 pub struct ReplicaConfig {
     pub replica_id: u64,
@@ -374,6 +385,10 @@ pub struct KeeperConfig {
 
     #[serde(default)]
     pub replica_config: Option<ReplicaConfig>,
+
+    /// Runtime configuration for the keeper service
+    #[serde(default)]
+    pub run_config: RunConfig,
 }
 
 // A secret is a string that can be provided either as a literal in the config,

apps/fortuna/src/keeper.rs

@@ -2,7 +2,7 @@ use {
     crate::{
         api::{BlockchainState, ChainId},
         chain::ethereum::{InstrumentedPythContract, InstrumentedSignablePythContract},
-        config::{EthereumConfig, ReplicaConfig},
+        config::{EthereumConfig, ReplicaConfig, RunConfig},
         eth_utils::traced_client::RpcMetrics,
         history::History,
         keeper::{
@@ -54,10 +54,12 @@ pub enum RequestState {
 
 /// Run threads to handle events for the last `BACKLOG_RANGE` blocks, watch for new blocks and
 /// handle any events for the new blocks.
+#[allow(clippy::too_many_arguments)] // Top level orchestration function that needs to configure several threads
 #[tracing::instrument(name = "keeper", skip_all, fields(chain_id = chain_state.id))]
 pub async fn run_keeper_threads(
     keeper_private_key: String,
     keeper_replica_config: Option<ReplicaConfig>,
+    keeper_run_config: RunConfig,
     chain_eth_config: EthereumConfig,
     chain_state: BlockchainState,
     metrics: Arc<KeeperMetrics>,
@@ -118,44 +120,52 @@ pub async fn run_keeper_threads(
     );
 
     // Spawn a thread that watches the keeper wallet balance and submits withdrawal transactions as needed to top-up the balance.
-    spawn(
-        withdraw_fees_wrapper(
-            contract.clone(),
-            chain_state.provider_address,
-            WITHDRAW_INTERVAL,
-            U256::from(chain_eth_config.min_keeper_balance),
-        )
-        .in_current_span(),
-    );
+    if !keeper_run_config.disable_fee_withdrawal {
+        spawn(
+            withdraw_fees_wrapper(
+                contract.clone(),
+                chain_state.provider_address,
+                WITHDRAW_INTERVAL,
+                U256::from(chain_eth_config.min_keeper_balance),
+            )
+            .in_current_span(),
+        );
+    } else {
+        tracing::info!("Fee withdrawal thread disabled by configuration");
+    }
 
     // Spawn a thread that periodically adjusts the provider fee.
-    spawn(
-        adjust_fee_wrapper(
-            contract.clone(),
-            chain_state.clone(),
-            chain_state.provider_address,
-            ADJUST_FEE_INTERVAL,
-            chain_eth_config.legacy_tx,
-            // NOTE: we are adjusting the fees based on the maximum configured gas for user transactions.
-            // However, the keeper will pad the gas limit for transactions (per the escalation policy) to ensure reliable submission.
-            // Consequently, fees can be adjusted such that transactions are still unprofitable.
-            // While we could scale up this value based on the padding, that ends up overcharging users as most transactions cost nowhere
-            // near the maximum gas limit.
-            // In the unlikely event that the keeper fees aren't sufficient, the solution to this is to configure the target
-            // fee percentage to be higher on that specific chain.
-            chain_eth_config.gas_limit,
-            // NOTE: unwrap() here so we panic early if someone configures these values below -100.
-            u64::try_from(100 + chain_eth_config.min_profit_pct)
-                .expect("min_profit_pct must be >= -100"),
-            u64::try_from(100 + chain_eth_config.target_profit_pct)
-                .expect("target_profit_pct must be >= -100"),
-            u64::try_from(100 + chain_eth_config.max_profit_pct)
-                .expect("max_profit_pct must be >= -100"),
-            chain_eth_config.fee,
-            metrics.clone(),
-        )
-        .in_current_span(),
-    );
+    if !keeper_run_config.disable_fee_adjustment {
+        spawn(
+            adjust_fee_wrapper(
+                contract.clone(),
+                chain_state.clone(),
+                chain_state.provider_address,
+                ADJUST_FEE_INTERVAL,
+                chain_eth_config.legacy_tx,
+                // NOTE: we are adjusting the fees based on the maximum configured gas for user transactions.
+                // However, the keeper will pad the gas limit for transactions (per the escalation policy) to ensure reliable submission.
+                // Consequently, fees can be adjusted such that transactions are still unprofitable.
+                // While we could scale up this value based on the padding, that ends up overcharging users as most transactions cost nowhere
+                // near the maximum gas limit.
+                // In the unlikely event that the keeper fees aren't sufficient, the solution to this is to configure the target
+                // fee percentage to be higher on that specific chain.
+                chain_eth_config.gas_limit,
+                // NOTE: unwrap() here so we panic early if someone configures these values below -100.
+                u64::try_from(100 + chain_eth_config.min_profit_pct)
+                    .expect("min_profit_pct must be >= -100"),
+                u64::try_from(100 + chain_eth_config.target_profit_pct)
+                    .expect("target_profit_pct must be >= -100"),
+                u64::try_from(100 + chain_eth_config.max_profit_pct)
+                    .expect("max_profit_pct must be >= -100"),
+                chain_eth_config.fee,
+                metrics.clone(),
+            )
+            .in_current_span(),
+        );
+    } else {
+        tracing::info!("Fee adjustment thread disabled by configuration");
+    }
 
     spawn(update_commitments_loop(contract.clone(), chain_state.clone()).in_current_span());