TQ: Async Nodes and P2P connections

Cargo.toml

@@ -143,6 +143,7 @@ members = [
     "test-utils",
     "trust-quorum",
     "trust-quorum/gfss",
+    "trust-quorum/protocol",
     "trust-quorum/test-utils",
     "trust-quorum/tqdb",
     "typed-rng",
@@ -304,6 +305,7 @@ default-members = [
     "sp-sim",
     "trust-quorum",
     "trust-quorum/gfss",
+    "trust-quorum/protocol",
     "trust-quorum/test-utils",
     "trust-quorum/tqdb",
     "test-utils",
@@ -370,6 +372,7 @@ assert_matches = "1.5.0"
 assert_cmd = "2.0.17"
 async-bb8-diesel = "0.2"
 async-trait = "0.1.89"
+attest-mock = { git = "https://github.com/oxidecomputer/dice-util", rev = "10952e8d9599b735b85d480af3560a11700e5b64" }
 atomicwrites = "0.4.4"
 authz-macros = { path = "nexus/authz-macros" }
 backoff = { version = "0.4.0", features = [ "tokio" ] }
@@ -471,6 +474,7 @@ gateway-types = { path = "gateway-types" }
 gethostname = "0.5.0"
 gfss = { path = "trust-quorum/gfss" }
 trust-quorum = { path = "trust-quorum" }
+trust-quorum-protocol = { path = "trust-quorum/protocol" }
 trust-quorum-test-utils = { path = "trust-quorum/test-utils" }
 glob = "0.3.2"
 guppy = "0.17.20"
@@ -724,7 +728,8 @@ slog-term = "2.9.1"
 smf = "0.2"
 socket2 = { version = "0.5", features = ["all"] }
 sp-sim = { path = "sp-sim" }
-sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "7da1f0b5dcd3d631da18b43ba78a84b1a2b425ee" }
+sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "dea3bbfac7d9d3c45f088898fcd05ee5d2ec2210" }
+sprockets-tls-test-utils = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "dea3bbfac7d9d3c45f088898fcd05ee5d2ec2210" }
 sqlformat = "0.3.5"
 sqlparser = { version = "0.45.0", features = [ "visitor" ] }
 static_assertions = "1.1.0"

sled-agent/src/bootstrap/config.rs

@@ -7,3 +7,4 @@
 pub const BOOTSTRAP_AGENT_HTTP_PORT: u16 = 80;
 pub const BOOTSTRAP_AGENT_RACK_INIT_PORT: u16 = 12346;
 pub const BOOTSTORE_PORT: u16 = 12347;
+pub const TRUST_QUORUM_PORT: u16 = 12349;

trust-quorum/Cargo.toml

@@ -3,18 +3,21 @@ name = "trust-quorum"
 version = "0.1.0"
 edition = "2021"
 license = "MPL-2.0"
+description = "trust quorum library for use by bootstrap agent"
 
 [lints]
 workspace = true
 
 [dependencies]
 anyhow.workspace = true
-bcs.workspace = true
 bootstore.workspace = true
+bytes.workspace = true
 camino.workspace = true
 chacha20poly1305.workspace = true
+ciborium.workspace = true
 daft.workspace = true
 derive_more.workspace = true
+futures.workspace = true
 gfss.workspace = true
 hex.workspace = true
 hkdf.workspace = true
@@ -28,29 +31,23 @@ sha3.workspace = true
 sled-agent-types.workspace = true
 slog.workspace = true
 slog-error-chain.workspace = true
+sprockets-tls.workspace = true
 static_assertions.workspace = true
 subtle.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
+trust-quorum-protocol.workspace = true
 uuid.workspace = true
 zeroize.workspace = true
 omicron-workspace-hack.workspace = true
 
 [dev-dependencies]
 assert_matches.workspace = true
+attest-mock.workspace = true
 dropshot.workspace = true
 omicron-test-utils.workspace = true
 proptest.workspace = true
 serde_json.workspace = true
 test-strategy.workspace = true
 trust-quorum-test-utils.workspace = true
-
-[features]
-# Impl `PartialEq` and `Eq` for types implementing `subtle::ConstantTimeEq` when
-# this feature is enabled.
-# 
-# This is of unknown risk. The rust compiler may obviate the security of using
-# subtle when we do this. On the other hand its very useful for testing and
-# debugging outside of production.
-danger_partial_eq_ct_wrapper = ["gfss/danger_partial_eq_ct_wrapper"]
-testing = []
+sprockets-tls-test-utils.workspace = true

trust-quorum/protocol/Cargo.toml

@@ -0,0 +1,57 @@
+[package]
+name = "trust-quorum-protocol"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+description = "sans-io trust quorum protocol implementation"
+
+[lints]
+workspace = true
+
+[dependencies]
+bootstore.workspace = true
+bytes.workspace = true
+camino.workspace = true
+chacha20poly1305.workspace = true
+ciborium.workspace = true
+daft.workspace = true
+derive_more.workspace = true
+gfss.workspace = true
+hex.workspace = true
+hkdf.workspace = true
+iddqd.workspace = true
+omicron-uuid-kinds.workspace = true
+rand = { workspace = true, features = ["os_rng"] }
+secrecy.workspace = true
+serde.workspace = true
+serde_with.workspace = true
+sha3.workspace = true
+sled-agent-types.workspace = true
+slog.workspace = true
+slog-error-chain.workspace = true
+static_assertions.workspace = true
+subtle.workspace = true
+thiserror.workspace = true
+uuid.workspace = true
+zeroize.workspace = true
+omicron-workspace-hack.workspace = true
+
+[dev-dependencies]
+assert_matches.workspace = true
+attest-mock.workspace = true
+dropshot.workspace = true
+omicron-test-utils.workspace = true
+proptest.workspace = true
+serde_json.workspace = true
+test-strategy.workspace = true
+trust-quorum-test-utils.workspace = true
+
+[features]
+# Impl `PartialEq` and `Eq` for types implementing `subtle::ConstantTimeEq` when
+# this feature is enabled.
+# 
+# This is of unknown risk. The rust compiler may obviate the security of using
+# subtle when we do this. On the other hand its very useful for testing and
+# debugging outside of production.
+danger_partial_eq_ct_wrapper = ["gfss/danger_partial_eq_ct_wrapper"]
+testing = []

trust-quorum/protocol/src/lib.rs

@@ -0,0 +1,162 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Implementation of the oxide rack trust quorum protocol
+//!
+//! This protocol is written as a
+//! [no-IO](https://sans-io.readthedocs.io/how-to-sans-io.html) implementation.
+//! All persistent state and all networking is managed outside of this
+//! implementation.
+
+use crypto::Sha3_256Digest;
+use daft::Diffable;
+use derive_more::Display;
+use gfss::shamir::Share;
+use serde::{Deserialize, Serialize};
+pub use sled_agent_types::sled::BaseboardId;
+use slog::{Logger, error, warn};
+
+mod alarm;
+mod compute_key_share;
+mod configuration;
+mod coordinator_state;
+pub(crate) mod crypto;
+mod messages;
+mod node;
+mod node_ctx;
+mod persistent_state;
+#[allow(unused)]
+mod rack_secret_loader;
+mod validators;
+
+pub use configuration::Configuration;
+pub use coordinator_state::{
+    CoordinatingMsg, CoordinatorOperation, CoordinatorState,
+    CoordinatorStateDiff,
+};
+pub use rack_secret_loader::{LoadRackSecretError, RackSecretLoaderDiff};
+pub use validators::{
+    ValidatedLrtqUpgradeMsgDiff, ValidatedReconfigureMsgDiff,
+};
+
+pub use alarm::Alarm;
+pub use crypto::RackSecret;
+pub use messages::*;
+pub use node::{Node, NodeDiff};
+// public only for docs.
+pub use node_ctx::NodeHandlerCtx;
+pub use node_ctx::{NodeCallerCtx, NodeCommonCtx, NodeCtx, NodeCtxDiff};
+pub use persistent_state::{
+    ExpungedMetadata, PersistentState, PersistentStateSummary,
+};
+
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    Deserialize,
+    Display,
+    Diffable,
+)]
+#[daft(leaf)]
+pub struct Epoch(pub u64);
+
+impl Epoch {
+    pub fn next(&self) -> Epoch {
+        Epoch(self.0.checked_add(1).expect("fewer than 2^64 epochs"))
+    }
+}
+
+/// The number of shares required to reconstruct the rack secret
+///
+/// Typically referred to as `k` in the docs
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Serialize,
+    Deserialize,
+    Display,
+    Diffable,
+)]
+#[daft(leaf)]
+pub struct Threshold(pub u8);
+
+/// A container to make messages between trust quorum nodes routable
+#[derive(Debug, Clone, Serialize, Deserialize, Diffable)]
+#[cfg_attr(feature = "danger_partial_eq_ct_wrapper", derive(PartialEq, Eq))]
+#[daft(leaf)]
+pub struct Envelope {
+    pub to: BaseboardId,
+    pub from: BaseboardId,
+    pub msg: PeerMsg,
+}
+
+#[cfg(feature = "testing")]
+impl Envelope {
+    pub fn equal_except_for_crypto_data(&self, other: &Self) -> bool {
+        self.to == other.to
+            && self.from == other.from
+            && self.msg.equal_except_for_crypto_data(&other.msg)
+    }
+}
+
+/// Check if a received share is valid for a given configuration
+///
+/// Return true if valid, false otherwise.
+pub fn validate_share(
+    log: &Logger,
+    config: &Configuration,
+    from: &BaseboardId,
+    epoch: Epoch,
+    share: &Share,
+) -> bool {
+    // Are we trying to retrieve shares for `epoch`?
+    if epoch != config.epoch {
+        warn!(
+            log,
+            "Received Share from node with wrong epoch";
+            "received_epoch" => %epoch,
+            "from" => %from
+        );
+        return false;
+    }
+
+    // Is the sender a member of the configuration `epoch`?
+    // Was the sender a member of the configuration at `old_epoch`?
+    let Some(expected_digest) = config.members.get(&from) else {
+        warn!(
+            log,
+            "Received Share from unexpected node";
+            "epoch" => %epoch,
+            "from" => %from
+        );
+        return false;
+    };
+
+    // Does the share hash match what we expect?
+    let mut digest = Sha3_256Digest::default();
+    share.digest::<sha3::Sha3_256>(&mut digest.0);
+    if digest != *expected_digest {
+        error!(
+            log,
+            "Received share with invalid digest";
+            "epoch" => %epoch,
+            "from" => %from
+        );
+        return false;
+    }
+
+    true
+}

trust-quorum/tests/cluster.rs → trust-quorum/protocol/tests/cluster.rs

@@ -15,7 +15,7 @@ use secrecy::ExposeSecret;
 use slog::{Logger, info, o};
 use std::collections::BTreeSet;
 use test_strategy::{Arbitrary, proptest};
-use trust_quorum::{
+use trust_quorum_protocol::{
     BaseboardId, CoordinatorOperation, Epoch, NodeCallerCtx, NodeCommonCtx,
     Threshold,
 };