use new ConfigReconcilerSpawnToken interface

Author: jgallagher

sled-agent/src/bootstrap/pre_server.rs

@@ -35,6 +35,7 @@ use illumos_utils::zone::Api;
 use illumos_utils::zone::Zones;
 use omicron_common::FileKv;
 use omicron_common::address::Ipv6Subnet;
+use sled_agent_config_reconciler::ConfigReconcilerSpawnToken;
 use sled_hardware::DendriteAsic;
 use sled_hardware::SledMode;
 use sled_hardware::underlay;
@@ -53,6 +54,7 @@ pub(super) struct BootstrapAgentStartup {
     pub(super) service_manager: ServiceManager,
     pub(super) long_running_task_handles: LongRunningTaskHandles,
     pub(super) sled_agent_started_tx: oneshot::Sender<SledAgent>,
+    pub(super) config_reconciler_spawn_token: ConfigReconcilerSpawnToken,
 }
 
 impl BootstrapAgentStartup {
@@ -118,6 +120,7 @@ impl BootstrapAgentStartup {
         // the process and are used by both the bootstrap agent and sled agent
         let (
             long_running_task_handles,
+            config_reconciler_spawn_token,
             sled_agent_started_tx,
             service_manager_ready_tx,
         ) = spawn_all_longrunning_tasks(
@@ -159,6 +162,7 @@ impl BootstrapAgentStartup {
             service_manager,
             long_running_task_handles,
             sled_agent_started_tx,
+            config_reconciler_spawn_token,
         })
     }
 }

sled-agent/src/bootstrap/server.rs

@@ -39,6 +39,7 @@ use omicron_ddm_admin_client::DdmError;
 use omicron_ddm_admin_client::types::EnableStatsRequest;
 use omicron_uuid_kinds::GenericUuid;
 use omicron_uuid_kinds::RackInitUuid;
+use sled_agent_config_reconciler::ConfigReconcilerSpawnToken;
 use sled_agent_config_reconciler::InternalDisksReceiver;
 use sled_agent_types::rack_init::RackInitializeRequest;
 use sled_agent_types::sled::StartSledAgentRequest;
@@ -180,6 +181,7 @@ impl Server {
             service_manager,
             long_running_task_handles,
             sled_agent_started_tx,
+            config_reconciler_spawn_token,
         } = BootstrapAgentStartup::run(config).await?;
 
         // Do we have a StartSledAgentRequest stored in the ledger?
@@ -246,6 +248,7 @@ impl Server {
                 &config,
                 start_sled_agent_request,
                 long_running_task_handles.clone(),
+                config_reconciler_spawn_token,
                 service_manager.clone(),
                 &base_log,
                 &startup_log,
@@ -261,7 +264,10 @@ impl Server {
 
             SledAgentState::ServerStarted(sled_agent_server)
         } else {
-            SledAgentState::Bootstrapping(Some(sled_agent_started_tx))
+            SledAgentState::Bootstrapping(Some(BootstrappingDependencies {
+                sled_agent_started_tx,
+                config_reconciler_spawn_token,
+            }))
         };
 
         // Spawn our inner task that handles any future hardware updates and any
@@ -303,11 +309,16 @@ impl Server {
 // bootstrap server).
 enum SledAgentState {
     // We're still in the bootstrapping phase, waiting for a sled-agent request.
-    Bootstrapping(Option<oneshot::Sender<SledAgent>>),
+    Bootstrapping(Option<BootstrappingDependencies>),
     // ... or the sled agent server is running.
     ServerStarted(SledAgentServer),
 }
 
+struct BootstrappingDependencies {
+    sled_agent_started_tx: oneshot::Sender<SledAgent>,
+    config_reconciler_spawn_token: ConfigReconcilerSpawnToken,
+}
+
 #[derive(thiserror::Error, Debug)]
 pub enum SledAgentServerStartError {
     #[error("Failed to start sled-agent server: {0}")]
@@ -347,6 +358,7 @@ async fn start_sled_agent(
     config: &SledConfig,
     request: StartSledAgentRequest,
     long_running_task_handles: LongRunningTaskHandles,
+    config_reconciler_spawn_token: ConfigReconcilerSpawnToken,
     service_manager: ServiceManager,
     base_log: &Logger,
     log: &Logger,
@@ -398,6 +410,7 @@ async fn start_sled_agent(
         base_log.clone(),
         request.clone(),
         long_running_task_handles.clone(),
+        config_reconciler_spawn_token,
         service_manager,
     )
     .await
@@ -531,7 +544,7 @@ impl Inner {
         log: &Logger,
     ) {
         match &mut self.state {
-            SledAgentState::Bootstrapping(sled_agent_started_tx) => {
+            SledAgentState::Bootstrapping(deps) => {
                 let request_id = request.body.id.into_untyped_uuid();
 
                 // Extract from options to satisfy the borrow checker.
@@ -540,13 +553,16 @@ impl Inner {
                 // we explicitly unwrap here, and panic on error below.
                 //
                 // See https://github.com/oxidecomputer/omicron/issues/4494
-                let sled_agent_started_tx =
-                    sled_agent_started_tx.take().unwrap();
+                let BootstrappingDependencies {
+                    sled_agent_started_tx,
+                    config_reconciler_spawn_token,
+                } = deps.take().unwrap();
 
                 let response = match start_sled_agent(
                     &self.config,
                     request,
                     self.long_running_task_handles.clone(),
+                    config_reconciler_spawn_token,
                     self.service_manager.clone(),
                     &self.base_log,
                     &log,

sled-agent/src/long_running_tasks.rs

@@ -24,7 +24,8 @@ use crate::zone_bundle::ZoneBundler;
 use bootstore::schemes::v0 as bootstore;
 use key_manager::{KeyManager, StorageKeyRequester};
 use sled_agent_config_reconciler::{
-    ConfigReconcilerHandle, RawDisksSender, TimeSyncConfig,
+    ConfigReconcilerHandle, ConfigReconcilerSpawnToken, RawDisksSender,
+    TimeSyncConfig,
 };
 use sled_agent_types::zone_bundle::CleanupContext;
 use sled_hardware::{HardwareManager, SledMode, UnparsedDisk};
@@ -61,6 +62,7 @@ pub async fn spawn_all_longrunning_tasks(
     config: &Config,
 ) -> (
     LongRunningTaskHandles,
+    ConfigReconcilerSpawnToken,
     oneshot::Sender<SledAgent>,
     oneshot::Sender<ServiceManager>,
 ) {
@@ -71,12 +73,13 @@ pub async fn spawn_all_longrunning_tasks(
     } else {
         TimeSyncConfig::Normal
     };
-    let mut config_reconciler = ConfigReconcilerHandle::new(
-        MountConfig::default(),
-        storage_key_requester,
-        time_sync_config,
-        log,
-    );
+    let (mut config_reconciler, config_reconciler_spawn_token) =
+        ConfigReconcilerHandle::new(
+            MountConfig::default(),
+            storage_key_requester,
+            time_sync_config,
+            log,
+        );
 
     let nongimlet_observed_disks =
         config.nongimlet_observed_disks.clone().unwrap_or(vec![]);
@@ -114,6 +117,7 @@ pub async fn spawn_all_longrunning_tasks(
             bootstore,
             zone_bundler,
         },
+        config_reconciler_spawn_token,
         sled_agent_started_tx,
         service_manager_ready_tx,
     )

sled-agent/src/server.rs

@@ -12,6 +12,7 @@ use crate::nexus::make_nexus_client;
 use crate::services::ServiceManager;
 use internal_dns_resolver::Resolver;
 use omicron_uuid_kinds::SledUuid;
+use sled_agent_config_reconciler::ConfigReconcilerSpawnToken;
 use sled_agent_types::sled::StartSledAgentRequest;
 use slog::Logger;
 use std::net::SocketAddr;
@@ -39,6 +40,7 @@ impl Server {
         log: Logger,
         request: StartSledAgentRequest,
         long_running_tasks_handles: LongRunningTaskHandles,
+        config_reconciler_spawn_token: ConfigReconcilerSpawnToken,
         services: ServiceManager,
     ) -> Result<Server, String> {
         info!(log, "setting up sled agent server");
@@ -61,6 +63,7 @@ impl Server {
             request,
             services,
             long_running_tasks_handles,
+            config_reconciler_spawn_token,
         )
         .await
         .map_err(|e| e.to_string())?;

sled-agent/src/sled_agent.rs

@@ -54,9 +54,9 @@ use omicron_common::disk::M2Slot;
 use omicron_ddm_admin_client::Client as DdmAdminClient;
 use omicron_uuid_kinds::{GenericUuid, PropolisUuid, SledUuid};
 use sled_agent_config_reconciler::{
-    ConfigReconcilerHandle, InternalDisksReceiver, LedgerNewConfigError,
-    LedgerTaskError, ReconcilerInventory, SledAgentArtifactStore,
-    SledAgentFacilities, TimeSyncStatus,
+    ConfigReconcilerHandle, ConfigReconcilerSpawnToken, InternalDisksReceiver,
+    LedgerNewConfigError, LedgerTaskError, ReconcilerInventory,
+    SledAgentArtifactStore, SledAgentFacilities, TimeSyncStatus,
 };
 use sled_agent_types::disk::DiskStateRequested;
 use sled_agent_types::early_networking::EarlyNetworkConfig;
@@ -399,6 +399,7 @@ impl SledAgent {
         request: StartSledAgentRequest,
         services: ServiceManager,
         long_running_task_handles: LongRunningTaskHandles,
+        config_reconciler_spawn_token: ConfigReconcilerSpawnToken,
     ) -> Result<SledAgent, Error> {
         // Pass the "parent_log" to all subcomponents that want to set their own
         // "component" value.
@@ -606,7 +607,7 @@ impl SledAgent {
                 zone_bundler: long_running_task_handles.zone_bundler.clone(),
             },
             SledAgentArtifactStoreWrapper(Arc::clone(&artifact_store)),
-            &parent_log,
+            config_reconciler_spawn_token,
         );
 
         services