Add Measurement Corpus artifact

This will be installed for wicket via the installinator document.
Skip installing anything for the moment.

Author: labbott

Cargo.lock

@@ -647,6 +647,7 @@ rand_distr = "0.4.3"
 rand_seeder = "0.3.0"
 range-requests = { path = "range-requests" }
 ratatui = "0.29.0"
+rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", branch = "main" }
 rayon = "1.10"
 rcgen = "0.12.1"
 reconfigurator-cli = { path = "dev-tools/reconfigurator-cli" }
@@ -747,10 +748,10 @@ toml_edit = "0.22.27"
 tough = { version = "0.20.0", features = [ "http" ] }
 transceiver-controller = { git = "https://github.com/oxidecomputer/transceiver-control", features = [ "api-traits" ] }
 trybuild = "1.0.106"
-tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
-tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", features = ["proptest", "schemars"] }
-tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
-tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
+tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2" }
+tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2", features = ["proptest", "schemars"] }
+tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2" }
+tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2" }
 tui-tree-widget = "0.23.1"
 typed-rng = { path = "typed-rng" }
 typify = "0.3.0"

Cargo.toml

@@ -17,6 +17,7 @@ hex.workspace = true
 omicron-pins.workspace = true
 omicron-workspace-hack.workspace = true
 omicron-zone-package.workspace = true
+rats-corim.workspace = true
 reqwest.workspace = true
 semver.workspace = true
 serde.workspace = true

dev-tools/releng/Cargo.toml

@@ -38,6 +38,19 @@ pub(crate) async fn fetch_hubris_artifacts(
 
     fs::create_dir_all(&output_dir).await?;
 
+    // We need to remove our old downloaded corpus to make sure nothing else
+    // gets added to the repo unexpectedly. This should only really be a
+    // issue with local builds
+    if std::fs::exists(&output_dir.join("measurement_corpus"))
+        .context("failed to check `measurement_corpus`")?
+    {
+        std::fs::remove_dir_all(&output_dir.join("measurement_corpus"))
+            .context("failed to remove `measurement_corpus")?;
+    }
+    fs::create_dir_all(&output_dir.join("measurement_corpus"))
+        .await
+        .context("Failed to create `measurement_corpus`")?;
+
     // This could be parallelized with FuturesUnordered but in practice this
     // takes less time than OS builds.
 
@@ -106,6 +119,22 @@ pub(crate) async fn fetch_hubris_artifacts(
                     }
                 }
             }
+            if let Some(corpus) = hash_manifest.corpus {
+                let hash = match corpus {
+                    Source::File(file) => file.hash,
+                    _ => anyhow::bail!(
+                        "Unexpected file type: should be a single file, not an RoT"
+                    ),
+                };
+                let data =
+                    fetch_hash(&logger, base_url, &client, &hash).await?;
+                fs::write(
+                    output_dir.join("measurement_corpus").join(hash),
+                    data,
+                )
+                .await
+                .context("failed to write file {hash}")?;
+            }
         }
     }
 
@@ -160,6 +189,9 @@ async fn fetch_hash(
 struct Manifest {
     #[serde(rename = "artifact")]
     artifacts: HashMap<KnownArtifactKind, Vec<Artifact>>,
+    // Add a default for backwards compatibility
+    #[serde(rename = "measurement_corpus")]
+    corpus: Option<Source>,
 }
 
 #[derive(Deserialize)]

dev-tools/releng/src/hubris.rs

@@ -76,6 +76,46 @@ pub(crate) async fn build_tuf_repo(
         }
     }
 
+    let mut measurement_corpus = vec![];
+
+    for entry in std::fs::read_dir(
+        output_dir.join("hubris-staging").join("measurement_corpus"),
+    )
+    .context("failed to read `hubris-staging/measurement_corpus")?
+    {
+        let entry = entry?;
+
+        let corim = rats_corim::Corim::from_file(entry.path())?;
+
+        measurement_corpus.push(DeserializedArtifactData {
+            name: format!("staging-{}", corim.id),
+            version: ArtifactVersion::new(corim.get_version()?.clone())?,
+            source: DeserializedArtifactSource::File {
+                path: Utf8PathBuf::from_path_buf(entry.path()).unwrap(),
+            },
+        });
+    }
+
+    for entry in std::fs::read_dir(
+        output_dir.join("hubris-production").join("measurement_corpus"),
+    )
+    .context("failed to read `hubris-production/measurement_corpus")?
+    {
+        let entry = entry?;
+        let corim = rats_corim::Corim::from_file(entry.path())?;
+        measurement_corpus.push(DeserializedArtifactData {
+            name: format!("production-{}", corim.id),
+            version: ArtifactVersion::new(corim.get_version()?.clone())?,
+            source: DeserializedArtifactSource::File {
+                path: Utf8PathBuf::from_path_buf(entry.path()).unwrap(),
+            },
+        });
+    }
+
+    manifest
+        .artifacts
+        .insert(KnownArtifactKind::MeasurementCorpus, measurement_corpus);
+
     // Add the OS images.
     manifest.artifacts.insert(
         KnownArtifactKind::Host,
@@ -115,6 +155,7 @@ pub(crate) async fn build_tuf_repo(
                 .join(format!("{}.tar.gz", package)),
         });
     }
+
     manifest.artifacts.insert(
         KnownArtifactKind::ControlPlane,
         vec![DeserializedArtifactData {

dev-tools/releng/src/tuf.rs

@@ -279,6 +279,9 @@ impl InstallOpts {
                                         control_plane_hash =
                                             Some(artifact.hash);
                                     }
+                                    InstallinatorArtifactKind::MeasurementCorpus => {
+    // Skip doing anything with this for the moment
+},
                                 }
                             }
 

installinator/src/dispatch.rs

@@ -144,6 +144,7 @@ pub fn try_make_update_rot_bootloader(
                     | KnownArtifactKind::Trampoline
                     | KnownArtifactKind::ControlPlane
                     | KnownArtifactKind::Zone
+                    | KnownArtifactKind::MeasurementCorpus
                     | KnownArtifactKind::PscRot
                     | KnownArtifactKind::SwitchRot
                     | KnownArtifactKind::GimletSp

nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs

@@ -121,6 +121,7 @@ pub fn try_make_update_sp(
                     | KnownArtifactKind::InstallinatorDocument
                     | KnownArtifactKind::ControlPlane
                     | KnownArtifactKind::Zone
+                    | KnownArtifactKind::MeasurementCorpus
                     | KnownArtifactKind::PscRot
                     | KnownArtifactKind::SwitchRot
                     | KnownArtifactKind::GimletRotBootloader

nexus/reconfigurator/planning/src/mgs_updates/sp.rs

@@ -275,7 +275,7 @@ impl<'a> UpdatePlanBuilder<'a> {
                 )
                 .await
             }
-            KnownArtifactKind::Zone => {
+            KnownArtifactKind::Zone | KnownArtifactKind::MeasurementCorpus => {
                 // We don't currently support repos with already split-out
                 // zones.
                 self.add_unknown_artifact(artifact_id, artifact_hash, stream)
@@ -302,6 +302,7 @@ impl<'a> UpdatePlanBuilder<'a> {
             | KnownArtifactKind::InstallinatorDocument
             | KnownArtifactKind::ControlPlane
             | KnownArtifactKind::Zone
+            | KnownArtifactKind::MeasurementCorpus
             | KnownArtifactKind::PscRot
             | KnownArtifactKind::SwitchRot
             | KnownArtifactKind::GimletRotBootloader
@@ -397,6 +398,7 @@ impl<'a> UpdatePlanBuilder<'a> {
             | KnownArtifactKind::InstallinatorDocument
             | KnownArtifactKind::ControlPlane
             | KnownArtifactKind::Zone
+            | KnownArtifactKind::MeasurementCorpus
             | KnownArtifactKind::PscRot
             | KnownArtifactKind::SwitchRot
             | KnownArtifactKind::GimletSp
@@ -503,6 +505,7 @@ impl<'a> UpdatePlanBuilder<'a> {
             | KnownArtifactKind::InstallinatorDocument
             | KnownArtifactKind::ControlPlane
             | KnownArtifactKind::Zone
+            | KnownArtifactKind::MeasurementCorpus
             | KnownArtifactKind::PscSp
             | KnownArtifactKind::SwitchSp
             | KnownArtifactKind::GimletRotBootloader

update-common/src/artifacts/update_plan.rs

@@ -117,6 +117,7 @@ scopeguard = { version = "1.2.0" }
 semver = { version = "1.0.26", features = ["serde"] }
 serde = { version = "1.0.219", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.142", features = ["raw_value", "unbounded_depth"] }
+serde_with = { version = "3.14.0" }
 sha1 = { version = "0.10.6", features = ["oid"] }
 sha2 = { version = "0.10.9", features = ["oid"] }
 similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] }
@@ -254,6 +255,7 @@ scopeguard = { version = "1.2.0" }
 semver = { version = "1.0.26", features = ["serde"] }
 serde = { version = "1.0.219", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.142", features = ["raw_value", "unbounded_depth"] }
+serde_with = { version = "3.14.0" }
 sha1 = { version = "0.10.6", features = ["oid"] }
 sha2 = { version = "0.10.9", features = ["oid"] }
 similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] }