Add Measurement Corpus artifact

This will be installed for wicket via the installinator document.
Skip installing anything for the moment.

Author: labbott

Cargo.lock

@@ -647,6 +647,7 @@ rand_distr = "0.4.3"
 rand_seeder = "0.3.0"
 range-requests = { path = "range-requests" }
 ratatui = "0.29.0"
+rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", branch = "main" }
 rayon = "1.10"
 rcgen = "0.12.1"
 reconfigurator-cli = { path = "dev-tools/reconfigurator-cli" }
@@ -747,10 +748,10 @@ toml_edit = "0.22.27"
 tough = { version = "0.20.0", features = [ "http" ] }
 transceiver-controller = { git = "https://github.com/oxidecomputer/transceiver-control", features = [ "api-traits" ] }
 trybuild = "1.0.106"
-tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
-tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", features = ["proptest", "schemars"] }
-tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
-tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
+tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2" }
+tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2", features = ["proptest", "schemars"] }
+tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2" }
+tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2" }
 tui-tree-widget = "0.23.1"
 typed-rng = { path = "typed-rng" }
 typify = "0.3.0"

Cargo.toml

@@ -17,6 +17,7 @@ hex.workspace = true
 omicron-pins.workspace = true
 omicron-workspace-hack.workspace = true
 omicron-zone-package.workspace = true
+rats-corim.workspace = true
 reqwest.workspace = true
 semver.workspace = true
 serde.workspace = true

dev-tools/releng/Cargo.toml

@@ -38,6 +38,19 @@ pub(crate) async fn fetch_hubris_artifacts(
 
     fs::create_dir_all(&output_dir).await?;
 
+    // We need to remove our old downloaded corpus to make sure nothing else
+    // gets added to the repo unexpectedly. This should only really be a
+    // issue with local builds
+    if std::fs::exists(&output_dir.join("measurement_corpus"))
+        .context("failed to check `measurement_corpus`")?
+    {
+        std::fs::remove_dir_all(&output_dir.join("measurement_corpus"))
+            .context("failed to remove `measurement_corpus")?;
+    }
+    fs::create_dir_all(&output_dir.join("measurement_corpus"))
+        .await
+        .context("Failed to create `measurement_corpus`")?;
+
     // This could be parallelized with FuturesUnordered but in practice this
     // takes less time than OS builds.
 
@@ -106,6 +119,22 @@ pub(crate) async fn fetch_hubris_artifacts(
                     }
                 }
             }
+            if let Some(corpus) = hash_manifest.corpus {
+                let hash = match corpus {
+                    Source::File(file) => file.hash,
+                    _ => anyhow::bail!(
+                        "Unexpected file type: should be a single file, not an RoT"
+                    ),
+                };
+                let data =
+                    fetch_hash(&logger, base_url, &client, &hash).await?;
+                fs::write(
+                    output_dir.join("measurement_corpus").join(hash),
+                    data,
+                )
+                .await
+                .context("failed to write file {hash}")?;
+            }
         }
     }
 
@@ -160,6 +189,9 @@ async fn fetch_hash(
 struct Manifest {
     #[serde(rename = "artifact")]
     artifacts: HashMap<KnownArtifactKind, Vec<Artifact>>,
+    // Add a default for backwards compatibility
+    #[serde(rename = "measurement_corpus")]
+    corpus: Option<Source>,
 }
 
 #[derive(Deserialize)]

dev-tools/releng/src/hubris.rs

@@ -76,6 +76,46 @@ pub(crate) async fn build_tuf_repo(
         }
     }
 
+    let mut measurement_corpus = vec![];
+
+    for entry in std::fs::read_dir(
+        output_dir.join("hubris-staging").join("measurement_corpus"),
+    )
+    .context("failed to read `hubris-staging/measurement_corpus")?
+    {
+        let entry = entry?;
+
+        let corim = rats_corim::Corim::from_file(entry.path())?;
+
+        measurement_corpus.push(DeserializedArtifactData {
+            name: format!("staging-{}", corim.id),
+            version: ArtifactVersion::new(corim.get_version()?.clone())?,
+            source: DeserializedArtifactSource::File {
+                path: Utf8PathBuf::from_path_buf(entry.path()).unwrap(),
+            },
+        });
+    }
+
+    for entry in std::fs::read_dir(
+        output_dir.join("hubris-production").join("measurement_corpus"),
+    )
+    .context("failed to read `hubris-production/measurement_corpus")?
+    {
+        let entry = entry?;
+        let corim = rats_corim::Corim::from_file(entry.path())?;
+        measurement_corpus.push(DeserializedArtifactData {
+            name: format!("production-{}", corim.id),
+            version: ArtifactVersion::new(corim.get_version()?.clone())?,
+            source: DeserializedArtifactSource::File {
+                path: Utf8PathBuf::from_path_buf(entry.path()).unwrap(),
+            },
+        });
+    }
+
+    manifest
+        .artifacts
+        .insert(KnownArtifactKind::MeasurementCorpus, measurement_corpus);
+
     // Add the OS images.
     manifest.artifacts.insert(
         KnownArtifactKind::Host,
@@ -115,6 +155,7 @@ pub(crate) async fn build_tuf_repo(
                 .join(format!("{}.tar.gz", package)),
         });
     }
+
     manifest.artifacts.insert(
         KnownArtifactKind::ControlPlane,
         vec![DeserializedArtifactData {

dev-tools/releng/src/tuf.rs

@@ -52,10 +52,13 @@ pub enum InstallinatorComponent {
     /// The control plane component.
     ControlPlane,
 
-    /// A component that means "both the host and the control plane", used for
-    /// writes for now. It is possible that this component will go away in the
-    /// future.
-    Both,
+    /// The measurement corpus component
+    MeasurementCorpus,
+
+    /// A component that means "all possible components (host phase 2, control
+    /// plane, and measurements)", used for writes for now. It is possible
+    /// that this component will go away in the future.
+    All,
 
     /// Future variants that might be unknown.
     #[serde(other, deserialize_with = "deserialize_ignore_any")]
@@ -194,6 +197,9 @@ pub enum WriteComponent {
     /// The control plane component.
     ControlPlane,
 
+    /// The measurement corpus
+    MeasurementCorpus,
+
     /// Future variants that might be unknown.
     #[serde(other, deserialize_with = "deserialize_ignore_any")]
     Unknown,
@@ -205,6 +211,7 @@ impl fmt::Display for WriteComponent {
             Self::HostPhase2 => f.write_str("host phase 2"),
             Self::ControlPlane => f.write_str("control plane"),
             Self::Unknown => f.write_str("unknown"),
+            Self::MeasurementCorpus => f.write_str("measurement corpus"),
         }
     }
 }
@@ -254,6 +261,8 @@ pub enum WriteError {
         #[source]
         error: Box<NestedEngineError<ControlPlaneZonesSpec>>,
     },
+    #[error("error creating directory: {error}")]
+    CreateDirError { error: std::io::Error },
 }
 
 impl From<NestedEngineError<ControlPlaneZonesSpec>> for WriteError {
@@ -303,6 +312,12 @@ pub enum ControlPlaneZonesStepId {
     /// Writing the zone manifest.
     ZoneManifest,
 
+    /// Writing the measurement corpus.
+    Measurement { name: String },
+
+    /// Ensure the measurement directory exists
+    CreateMeasurementDir,
+
     /// Syncing writes to disk.
     Fsync,
 

installinator-common/src/progress.rs

@@ -111,11 +111,23 @@ pub(crate) enum LookupIdKind {
     Hashes { host_phase_2: ArtifactHash, control_plane: ArtifactHash },
 }
 
+#[derive(Clone, Debug)]
+pub(crate) struct MeasurementArtifact {
+    pub(crate) hash: ArtifactHash,
+    pub(crate) name: String,
+}
+
+pub(crate) struct MeasurementArtifactHashId {
+    pub(crate) hash: ArtifactHashId,
+    pub(crate) name: String,
+}
+
 /// The host phase 2 and control plane hashes to download.
 #[derive(Clone, Debug)]
 pub(crate) struct ArtifactsToDownload {
     pub(crate) host_phase_2: ArtifactHash,
     pub(crate) control_plane: ArtifactHash,
+    pub(crate) measurement_corpus: Vec<MeasurementArtifact>,
 }
 
 impl ArtifactsToDownload {
@@ -132,6 +144,20 @@ impl ArtifactsToDownload {
             hash: self.control_plane,
         }
     }
+
+    pub(crate) fn measurement_corpus(&self) -> Vec<MeasurementArtifactHashId> {
+        self.measurement_corpus
+            .clone()
+            .into_iter()
+            .map(|x| MeasurementArtifactHashId {
+                name: x.name,
+                hash: ArtifactHashId {
+                    kind: KnownArtifactKind::MeasurementCorpus.into(),
+                    hash: x.hash,
+                },
+            })
+            .collect()
+    }
 }
 
 #[derive(Debug)]