From 5be9d255e62643f483f1840d6bd147989e0c29e8 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 3 Sep 2026 02:21:27 +0800 Subject: [PATCH] Fix rustfs-cli e2e target warning (#7073) * fix: split rustfs-cli e2e entrypoint * fix: allow e2e enrollment feature without fixtures --- .config/nextest.toml | 4 +- Cargo.lock | 58 ++- Cargo.toml | 2 +- rustfs/Cargo.toml | 2 +- rustfs/src/bin/common/mod.rs | 525 +++++++++++++++++++++++ rustfs/src/bin/rustfs-cli-e2e.rs | 21 + rustfs/src/bin/rustfs-cli.rs | 508 +--------------------- rustfs/src/connect/offline/enrollment.rs | 21 +- 8 files changed, 615 insertions(+), 526 deletions(-) create mode 100644 rustfs/src/bin/common/mod.rs create mode 100644 rustfs/src/bin/rustfs-cli-e2e.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 8c08956fc..32adfa6bb 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.default.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.default.scripts]] @@ -210,7 +210,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.ci.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.ci.scripts]] diff --git a/Cargo.lock b/Cargo.lock index bb9b2b595..8af2498c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,13 +125,28 @@ version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" +[[package]] +name = "alloc-no-stdlib" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" + [[package]] name = "alloc-stdlib" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" dependencies = [ - "alloc-no-stdlib", + "alloc-no-stdlib 2.0.4", +] + +[[package]] +name = "alloc-stdlib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" +dependencies = [ + "alloc-no-stdlib 3.0.0", ] [[package]] @@ -1728,9 +1743,20 @@ version = "8.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", + "alloc-no-stdlib 2.0.4", + "alloc-stdlib 0.2.4", + "brotli-decompressor 5.0.3", +] + +[[package]] +name = "brotli" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b" +dependencies = [ + "alloc-no-stdlib 3.0.0", + "alloc-stdlib 0.3.0", + "brotli-decompressor 6.0.0", ] [[package]] @@ -1739,8 +1765,18 @@ version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "alloc-no-stdlib 2.0.4", + "alloc-stdlib 0.2.4", +] + +[[package]] +name = "brotli-decompressor" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46087801e40b785410c352bee60e9b6226312709cf52d5fff92776cdf5cf6490" +dependencies = [ + "alloc-no-stdlib 3.0.0", + "alloc-stdlib 0.3.0", ] [[package]] @@ -2157,7 +2193,7 @@ version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ - "brotli", + "brotli 8.0.4", "bzip2", "compression-core", "flate2", @@ -6554,9 +6590,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -7540,7 +7576,7 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.23.1", - "brotli", + "brotli 8.0.4", "bytes", "chrono", "flate2", @@ -10855,7 +10891,7 @@ version = "1.0.0-rc.5" dependencies = [ "base64-simd", "blake2", - "brotli", + "brotli 9.0.0", "bytes", "convert_case 0.12.0", "crc-fast", diff --git a/Cargo.toml b/Cargo.toml index 03de7d352..271279430 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -247,7 +247,7 @@ aws-smithy-http-client = { default-features = false, version = "1.4.0" } aws-smithy-runtime-api = { version = "1.15.0" } aws-smithy-types = { version = "1.6.2" } base64-simd = "0.8.0" -brotli = "8.0.4" +brotli = "9.0.0" clap = { version = "4.6.6" } const-str = { version = "1.1.0" } convert_case = "0.12.0" diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 73db55970..d34f4d928 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -41,7 +41,7 @@ path = "src/bin/rustfs-cli.rs" [[bin]] name = "rustfs-cli-e2e" -path = "src/bin/rustfs-cli.rs" +path = "src/bin/rustfs-cli-e2e.rs" required-features = ["offline-enrollment-e2e-root"] # The Swift protocol suites are manual (#[ignore]) and need a server built with diff --git a/rustfs/src/bin/common/mod.rs b/rustfs/src/bin/common/mod.rs new file mode 100644 index 000000000..4cdf9eda2 --- /dev/null +++ b/rustfs/src/bin/common/mod.rs @@ -0,0 +1,525 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Offline and diagnostic RustFS command-line entry point. +//! +//! This binary shares RustFS's existing subcommand dispatcher and provides the +//! documented entry point for offline tooling such as `inspect bucket-meta`. + +use std::fs; +use std::future::Future; +use std::io::{Read as _, Write as _}; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::process::ExitCode; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use rustfs::connect::offline::{ + BundleContext, BundleError, BundleReceipt, OfflineEnrollment, OfflineKeyStore, collect_offline_diagnostics, + write_offline_bundle, +}; +use tokio_util::sync::CancellationToken; + +/// Owner read/write only. The response names the key being enrolled and the +/// challenge it answers; neither belongs to anyone else on the machine. +#[cfg(unix)] +const RESPONSE_MODE: u32 = 0o600; +const OFFLINE_RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100); + +const USAGE: &str = "\ +Usage: rustfs-cli connect offline enroll --challenge --output [--key-dir ] + rustfs-cli connect offline bundle --state-dir --device-name --output [--key-dir ] + +Answers a Connect offline enrolment challenge without a network. Reads the +challenge from a file or from stdin when the path is `-`, verifies it against the +enrolment root compiled into this binary, mints the key being enrolled on first +use, and writes the signed response. + +Builds a deterministic support bundle from the stopped runtime's persisted +inventory and bounded host diagnostics. The bundle command never uploads. + +No secret is ever accepted on the command line. +"; + +pub fn run() -> ExitCode { + let arguments: Vec = std::env::args().skip(1).collect(); + + // Offline operations are handled before the server dispatcher so they can + // never enter the networked server runtime. Bundle collection creates only + // a current-thread runtime for its timeout and SIGINT cancellation. + if matches!( + arguments.first().map(String::as_str), + Some("connect") if matches!(arguments.get(1).map(String::as_str), Some("offline")) + ) { + return match run_offline(&arguments[2..]) { + Ok(()) => ExitCode::SUCCESS, + Err(message) => { + eprintln!("rustfs-cli: {message}"); + ExitCode::FAILURE + } + }; + } + + rustfs::startup_entrypoint::run_process(); + + ExitCode::SUCCESS +} + +fn run_offline(arguments: &[String]) -> Result<(), String> { + match arguments.first().map(String::as_str) { + Some("enroll") => enroll(&arguments[1..]), + Some("bundle") => bundle(&arguments[1..]), + Some(other) => Err(format!("unknown offline subcommand `{other}`\n\n{USAGE}")), + None => Err(format!("missing offline subcommand\n\n{USAGE}")), + } +} + +fn bundle(arguments: &[String]) -> Result<(), String> { + if !cfg!(target_os = "linux") { + return Err(BundleError::UnsupportedPlatform.to_string()); + } + + let mut state_directory: Option = None; + let mut device_name: Option = None; + let mut output_path: Option = None; + let mut key_directory: Option = None; + + let mut index = 0; + while index < arguments.len() { + let flag = arguments[index].as_str(); + let take_value = |name: &str| -> Result { + arguments + .get(index + 1) + .cloned() + .ok_or_else(|| format!("`{name}` needs a value\n\n{USAGE}")) + }; + + match flag { + "--state-dir" => state_directory = Some(take_value("--state-dir")?), + "--device-name" => device_name = Some(take_value("--device-name")?), + "--output" => output_path = Some(take_value("--output")?), + "--key-dir" => key_directory = Some(take_value("--key-dir")?), + "-h" | "--help" => { + println!("{USAGE}"); + return Ok(()); + } + other => return Err(format!("unknown option `{other}`\n\n{USAGE}")), + } + + index += 2; + } + + let state_directory = state_directory.ok_or_else(|| format!("`--state-dir` is required\n\n{USAGE}"))?; + let device_name = device_name.ok_or_else(|| format!("`--device-name` is required\n\n{USAGE}"))?; + let output_path = output_path.ok_or_else(|| format!("`--output` is required\n\n{USAGE}"))?; + let key_directory = key_directory.unwrap_or_else(|| ".".to_owned()); + let key = OfflineKeyStore::new(&key_directory) + .load() + .map_err(|error| error.to_string())? + .ok_or_else(|| "offline enrollment key is missing; run `connect offline enroll` first".to_owned())?; + + let cancel = CancellationToken::new(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .enable_io() + .build() + .map_err(|error| format!("cannot create the offline collector runtime: {error}"))?; + let output = PathBuf::from(&output_path); + let operation = async move { + let signal = async { + tokio::signal::ctrl_c() + .await + .map_err(|error| format!("cannot listen for SIGINT: {error}")) + }; + tokio::pin!(signal); + let diagnostics = tokio::select! { + biased; + signal = signal.as_mut() => { + cancel.cancel(); + signal?; + return Err("offline bundle production was cancelled".to_owned()); + } + result = collect_offline_diagnostics(Path::new(&state_directory), &cancel) => { + result.map_err(|error| error.to_string())? + } + }; + + let elapsed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| "the system clock is before the Unix epoch".to_owned())?; + let produced_at_unix = + i64::try_from(elapsed.as_secs()).map_err(|_| "the system clock is outside the supported range".to_owned())?; + let timestamp_millis = + u64::try_from(elapsed.as_millis()).map_err(|_| "the system clock is outside the supported range".to_owned())?; + let mut nonce = [0u8; 32]; + getrandom(&mut nonce)?; + let mut uuid_random = [0u8; 10]; + getrandom(&mut uuid_random)?; + let bundle_uid = uuid::Builder::from_unix_timestamp_millis(timestamp_millis, &uuid_random) + .into_uuid() + .to_string(); + let inventory_captured_at = diagnostics.inventory_captured_at; + let inventory_age = diagnostics.inventory_age; + let writer_cancel = cancel.clone(); + let writer = tokio::task::spawn_blocking(move || { + write_offline_bundle( + &output, + &BundleContext { + bundle_uid, + device_name, + nonce, + produced_at_unix, + }, + &diagnostics.entries, + &key, + &writer_cancel, + ) + }); + let receipt = await_offline_writer(&cancel, signal.as_mut(), writer).await?; + Ok((receipt, inventory_captured_at, inventory_age)) + }; + let (runtime, (receipt, inventory_captured_at, inventory_age)) = run_offline_runtime(runtime, operation)?; + drop(runtime); + + println!("Bundle: {}", receipt.bundle_uid); + println!("Device: {}", receipt.device_name); + println!("Inventory: {} ({}s old)", inventory_captured_at, inventory_age.as_secs()); + println!("L0: {} entries, {} bytes", receipt.l0_count, receipt.l0_bytes); + println!("L1: {} entries, {} bytes", receipt.l1_count, receipt.l1_bytes); + println!( + "Redaction: {} ({})", + rustfs::connect::offline::redaction::REDACTION_VERSION, + rustfs::connect::offline::redaction::RULESET_HASH + ); + println!("Archive: {} bytes, sha256 {}", receipt.archive_size_bytes, receipt.archive_sha256); + println!("Output: {output_path}"); + println!("Upload: not performed"); + + Ok(()) +} + +async fn await_offline_writer( + cancel: &CancellationToken, + mut signal: Pin<&mut S>, + mut writer: tokio::task::JoinHandle>, +) -> Result +where + S: Future> + ?Sized, +{ + tokio::select! { + biased; + signal = signal.as_mut() => { + cancel.cancel(); + let writer = finish_offline_writer(writer.await); + match writer { + Ok(receipt) => Ok(receipt), + Err(error) => { + signal?; + Err(error) + } + } + } + result = &mut writer => finish_offline_writer(result), + } +} + +fn finish_offline_writer( + result: Result, tokio::task::JoinError>, +) -> Result { + result + .map_err(|error| format!("offline bundle writer task failed: {error}"))? + .map_err(|error| error.to_string()) +} + +fn run_offline_runtime( + runtime: tokio::runtime::Runtime, + operation: impl Future>, +) -> Result<(tokio::runtime::Runtime, T), String> { + match runtime.block_on(operation) { + Ok(value) => Ok((runtime, value)), + Err(error) => { + runtime.shutdown_timeout(OFFLINE_RUNTIME_SHUTDOWN_TIMEOUT); + Err(error) + } + } +} + +fn enroll(arguments: &[String]) -> Result<(), String> { + let mut challenge_path: Option = None; + let mut output_path: Option = None; + let mut key_directory: Option = None; + + let mut index = 0; + while index < arguments.len() { + let flag = arguments[index].as_str(); + let take_value = |name: &str| -> Result { + arguments + .get(index + 1) + .cloned() + .ok_or_else(|| format!("`{name}` needs a value\n\n{USAGE}")) + }; + + match flag { + "--challenge" => challenge_path = Some(take_value("--challenge")?), + "--output" => output_path = Some(take_value("--output")?), + "--key-dir" => key_directory = Some(take_value("--key-dir")?), + "-h" | "--help" => { + println!("{USAGE}"); + return Ok(()); + } + other => return Err(format!("unknown option `{other}`\n\n{USAGE}")), + } + + index += 2; + } + + let challenge_path = challenge_path.ok_or_else(|| format!("`--challenge` is required\n\n{USAGE}"))?; + let output_path = output_path.ok_or_else(|| format!("`--output` is required\n\n{USAGE}"))?; + let key_directory = key_directory.unwrap_or_else(|| ".".to_string()); + + let challenge = read_challenge(&challenge_path)?; + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| "the system clock is before the Unix epoch".to_string())? + .as_secs() as i64; + #[cfg(feature = "offline-enrollment-e2e-root")] + let now = enrollment_evaluation_time(now)?; + + #[cfg(feature = "offline-enrollment-e2e-root")] + let verified = verify_enrollment_challenge(&challenge, now).map_err(|error| error.to_string())?; + #[cfg(not(feature = "offline-enrollment-e2e-root"))] + let verified = OfflineEnrollment::verify_challenge(&challenge, now).map_err(|error| error.to_string())?; + + // First use mints the key; a retry answers with the one already enrolled, + // because the operator may already be carrying a response naming it. + let key = OfflineKeyStore::new(&key_directory) + .load_or_create() + .map_err(|error| error.to_string())?; + + let mut device_nonce = [0u8; 32]; + getrandom(&mut device_nonce)?; + + let response = OfflineEnrollment::build_response(&verified, &key, &device_nonce, now).map_err(|error| error.to_string())?; + + write_response(Path::new(&output_path), &response)?; + + Ok(()) +} + +#[cfg(feature = "offline-enrollment-e2e-root")] +fn enrollment_evaluation_time(system_now: i64) -> Result { + // The public fixture gate needs a compile-time clock; real E2E builds omit it. + if env!("CARGO_BIN_NAME") == "rustfs-cli-e2e" + && let Some(value) = option_env!("RUSTFS_E2E_OFFLINE_ENROLLMENT_FIXTURE_TIME") + { + return value + .parse() + .map_err(|_| "the compiled E2E enrollment fixture time is invalid".to_owned()); + } + + Ok(system_now) +} + +#[cfg(feature = "offline-enrollment-e2e-root")] +fn verify_enrollment_challenge( + challenge: &[u8], + now: i64, +) -> Result { + if env!("CARGO_BIN_NAME") == "rustfs-cli-e2e" { + return OfflineEnrollment::verify_e2e_challenge(challenge, now); + } + + OfflineEnrollment::verify_challenge(challenge, now) +} + +/// Reads the challenge from a file, or from stdin when the path is `-`. +/// +/// A challenge is not a secret — it is signed, public, and carried in by hand — +/// so accepting a path is safe. The response's key never arrives this way. +fn read_challenge(path: &str) -> Result, String> { + if path == "-" { + let mut buffer = Vec::new(); + std::io::stdin() + .read_to_end(&mut buffer) + .map_err(|error| format!("cannot read the challenge from stdin: {error}"))?; + return Ok(buffer); + } + + fs::read(path).map_err(|error| format!("cannot read the challenge at {path}: {error}")) +} + +/// Writes the response durably and atomically at mode 0600. +/// +/// Not the no-clobber publish `IdentityStore` performs for a key: an operator +/// who reruns an enrolment expects the response file to be replaced, whereas a +/// second key would strand the first. Same durability, deliberately different +/// publication rule. +fn write_response(path: &Path, response: &[u8]) -> Result<(), String> { + let parent = path.parent().filter(|parent| !parent.as_os_str().is_empty()); + let temporary: PathBuf = match parent { + Some(parent) => parent.join(format!(".{}.tmp", file_name(path))), + None => PathBuf::from(format!(".{}.tmp", file_name(path))), + }; + + let mut options = fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(RESPONSE_MODE); + } + + let write = (|| -> std::io::Result<()> { + let mut file = options.open(&temporary)?; + file.write_all(response)?; + + // The umask can only narrow the creation mode, so set the exact mode + // before the bytes become durable. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + file.set_permissions(fs::Permissions::from_mode(RESPONSE_MODE))?; + } + + file.sync_all() + })(); + + if let Err(error) = write { + let _ = fs::remove_file(&temporary); + return Err(format!("cannot write the response to {}: {error}", path.display())); + } + + fs::rename(&temporary, path).map_err(|error| { + let _ = fs::remove_file(&temporary); + format!("cannot publish the response at {}: {error}", path.display()) + })?; + + if let Some(parent) = parent { + sync_directory(parent); + } + + Ok(()) +} + +fn file_name(path: &Path) -> String { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_else(|| "response".to_string()) +} + +/// Fsync the directory so the renamed entry survives power loss. Directories +/// cannot be opened for syncing on Windows, where this is a no-op. +fn sync_directory(directory: &Path) { + #[cfg(unix)] + { + if let Ok(handle) = fs::File::open(directory) { + let _ = handle.sync_all(); + } + } + #[cfg(not(unix))] + let _ = directory; +} + +/// Fills `buffer` with operating-system randomness. +/// +/// The device nonce must be unpredictable: it is what stops a captured response +/// being replayed as a fresh one. Use the workspace rand 0.10 system RNG, +/// matching the rand_core version used by the upgraded crypto stack. +fn getrandom(buffer: &mut [u8]) -> Result<(), String> { + use rand::{TryRng as _, rngs::SysRng}; + + SysRng + .try_fill_bytes(buffer) + .map_err(|error| format!("the operating system random source failed: {error}")) +} + +#[cfg(test)] +mod tests { + use std::time::Instant; + + use super::*; + + #[cfg(feature = "offline-enrollment-e2e-root")] + #[test] + fn only_the_dedicated_e2e_target_selects_the_test_root() { + let challenge = + fs::read(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/offline-enrollment-e2e/challenge.json")) + .expect("read E2E enrollment challenge"); + let result = verify_enrollment_challenge(&challenge, 4_070_908_800); + + if env!("CARGO_BIN_NAME") == "rustfs-cli-e2e" { + result.expect("the dedicated E2E binary selects the test root"); + } else { + assert_eq!( + result + .expect_err("every production binary must retain the hosted root") + .reason(), + "ENROLLMENT_ROOT_UNKNOWN" + ); + } + } + + #[test] + fn connect_offline_bundle_cli_shutdown_is_bounded_with_a_stuck_blocking_collector() { + let runtime = tokio::runtime::Builder::new_current_thread().build().expect("test runtime"); + let operation = async { + let (started, observed) = tokio::sync::oneshot::channel(); + tokio::task::spawn_blocking(move || { + started.send(()).expect("report blocking task"); + std::thread::sleep(Duration::from_secs(5)); + }); + observed.await.expect("blocking task started"); + Err::<(), _>("collector cancelled".to_owned()) + }; + + let started_at = Instant::now(); + assert!(run_offline_runtime(runtime, operation).is_err()); + assert!(started_at.elapsed() < Duration::from_secs(1)); + } + + #[test] + fn connect_offline_bundle_cli_preserves_success_when_signal_and_committed_writer_are_ready() { + let runtime = tokio::runtime::Builder::new_current_thread().build().expect("test runtime"); + let cancel = CancellationToken::new(); + let temp = tempfile::tempdir().expect("output tempdir"); + let output = temp.path().join("bundle.zip"); + let writer_output = output.clone(); + let signal = std::future::ready(Ok(())); + tokio::pin!(signal); + let writer = runtime.spawn(async move { + fs::write(writer_output, b"published bundle").expect("publish test bundle"); + Ok(BundleReceipt { + bundle_uid: "0198f3a1-8000-7e50-8f61-4a5b6c7d8e94".to_owned(), + device_name: "organizations/0198f3a1-4c00-7a10-8b21-0c1d2e3f4a50/clusters/0198f3a1-5d00-7b20-9c31-1d2e3f4a5b61/clusterDevices/0198f3a1-6e00-7c30-ad41-2e3f4a5b6c72".to_owned(), + archive_size_bytes: 1, + archive_sha256: "00".repeat(32), + l0_count: 6, + l0_bytes: 1, + l1_count: 6, + l1_bytes: 1, + }) + }); + runtime.block_on(async { + while !writer.is_finished() { + tokio::task::yield_now().await; + } + }); + assert_eq!(fs::read(&output).expect("read committed output"), b"published bundle"); + + let result = runtime.block_on(await_offline_writer(&cancel, signal.as_mut(), writer)); + assert!(result.is_ok()); + assert!(cancel.is_cancelled()); + } +} diff --git a/rustfs/src/bin/rustfs-cli-e2e.rs b/rustfs/src/bin/rustfs-cli-e2e.rs new file mode 100644 index 000000000..972a0e5b2 --- /dev/null +++ b/rustfs/src/bin/rustfs-cli-e2e.rs @@ -0,0 +1,21 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod common; + +use std::process::ExitCode; + +fn main() -> ExitCode { + common::run() +} diff --git a/rustfs/src/bin/rustfs-cli.rs b/rustfs/src/bin/rustfs-cli.rs index f87c93bcd..972a0e5b2 100644 --- a/rustfs/src/bin/rustfs-cli.rs +++ b/rustfs/src/bin/rustfs-cli.rs @@ -12,514 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Offline and diagnostic RustFS command-line entry point. -//! -//! This binary shares RustFS's existing subcommand dispatcher and provides the -//! documented entry point for offline tooling such as `inspect bucket-meta`. +mod common; -use std::fs; -use std::future::Future; -use std::io::{Read as _, Write as _}; -use std::path::{Path, PathBuf}; -use std::pin::Pin; use std::process::ExitCode; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -use rustfs::connect::offline::{ - BundleContext, BundleError, BundleReceipt, OfflineEnrollment, OfflineKeyStore, collect_offline_diagnostics, - write_offline_bundle, -}; -use tokio_util::sync::CancellationToken; - -/// Owner read/write only. The response names the key being enrolled and the -/// challenge it answers; neither belongs to anyone else on the machine. -#[cfg(unix)] -const RESPONSE_MODE: u32 = 0o600; -const OFFLINE_RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100); - -const USAGE: &str = "\ -Usage: rustfs-cli connect offline enroll --challenge --output [--key-dir ] - rustfs-cli connect offline bundle --state-dir --device-name --output [--key-dir ] - -Answers a Connect offline enrolment challenge without a network. Reads the -challenge from a file or from stdin when the path is `-`, verifies it against the -enrolment root compiled into this binary, mints the key being enrolled on first -use, and writes the signed response. - -Builds a deterministic support bundle from the stopped runtime's persisted -inventory and bounded host diagnostics. The bundle command never uploads. - -No secret is ever accepted on the command line. -"; fn main() -> ExitCode { - let arguments: Vec = std::env::args().skip(1).collect(); - - // Offline operations are handled before the server dispatcher so they can - // never enter the networked server runtime. Bundle collection creates only - // a current-thread runtime for its timeout and SIGINT cancellation. - if matches!( - arguments.first().map(String::as_str), - Some("connect") if matches!(arguments.get(1).map(String::as_str), Some("offline")) - ) { - return match run_offline(&arguments[2..]) { - Ok(()) => ExitCode::SUCCESS, - Err(message) => { - eprintln!("rustfs-cli: {message}"); - ExitCode::FAILURE - } - }; - } - - rustfs::startup_entrypoint::run_process(); - - ExitCode::SUCCESS -} - -fn run_offline(arguments: &[String]) -> Result<(), String> { - match arguments.first().map(String::as_str) { - Some("enroll") => enroll(&arguments[1..]), - Some("bundle") => bundle(&arguments[1..]), - Some(other) => Err(format!("unknown offline subcommand `{other}`\n\n{USAGE}")), - None => Err(format!("missing offline subcommand\n\n{USAGE}")), - } -} - -fn bundle(arguments: &[String]) -> Result<(), String> { - if !cfg!(target_os = "linux") { - return Err(BundleError::UnsupportedPlatform.to_string()); - } - - let mut state_directory: Option = None; - let mut device_name: Option = None; - let mut output_path: Option = None; - let mut key_directory: Option = None; - - let mut index = 0; - while index < arguments.len() { - let flag = arguments[index].as_str(); - let take_value = |name: &str| -> Result { - arguments - .get(index + 1) - .cloned() - .ok_or_else(|| format!("`{name}` needs a value\n\n{USAGE}")) - }; - - match flag { - "--state-dir" => state_directory = Some(take_value("--state-dir")?), - "--device-name" => device_name = Some(take_value("--device-name")?), - "--output" => output_path = Some(take_value("--output")?), - "--key-dir" => key_directory = Some(take_value("--key-dir")?), - "-h" | "--help" => { - println!("{USAGE}"); - return Ok(()); - } - other => return Err(format!("unknown option `{other}`\n\n{USAGE}")), - } - - index += 2; - } - - let state_directory = state_directory.ok_or_else(|| format!("`--state-dir` is required\n\n{USAGE}"))?; - let device_name = device_name.ok_or_else(|| format!("`--device-name` is required\n\n{USAGE}"))?; - let output_path = output_path.ok_or_else(|| format!("`--output` is required\n\n{USAGE}"))?; - let key_directory = key_directory.unwrap_or_else(|| ".".to_owned()); - let key = OfflineKeyStore::new(&key_directory) - .load() - .map_err(|error| error.to_string())? - .ok_or_else(|| "offline enrollment key is missing; run `connect offline enroll` first".to_owned())?; - - let cancel = CancellationToken::new(); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_time() - .enable_io() - .build() - .map_err(|error| format!("cannot create the offline collector runtime: {error}"))?; - let output = PathBuf::from(&output_path); - let operation = async move { - let signal = async { - tokio::signal::ctrl_c() - .await - .map_err(|error| format!("cannot listen for SIGINT: {error}")) - }; - tokio::pin!(signal); - let diagnostics = tokio::select! { - biased; - signal = signal.as_mut() => { - cancel.cancel(); - signal?; - return Err("offline bundle production was cancelled".to_owned()); - } - result = collect_offline_diagnostics(Path::new(&state_directory), &cancel) => { - result.map_err(|error| error.to_string())? - } - }; - - let elapsed = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| "the system clock is before the Unix epoch".to_owned())?; - let produced_at_unix = - i64::try_from(elapsed.as_secs()).map_err(|_| "the system clock is outside the supported range".to_owned())?; - let timestamp_millis = - u64::try_from(elapsed.as_millis()).map_err(|_| "the system clock is outside the supported range".to_owned())?; - let mut nonce = [0u8; 32]; - getrandom(&mut nonce)?; - let mut uuid_random = [0u8; 10]; - getrandom(&mut uuid_random)?; - let bundle_uid = uuid::Builder::from_unix_timestamp_millis(timestamp_millis, &uuid_random) - .into_uuid() - .to_string(); - let inventory_captured_at = diagnostics.inventory_captured_at; - let inventory_age = diagnostics.inventory_age; - let writer_cancel = cancel.clone(); - let writer = tokio::task::spawn_blocking(move || { - write_offline_bundle( - &output, - &BundleContext { - bundle_uid, - device_name, - nonce, - produced_at_unix, - }, - &diagnostics.entries, - &key, - &writer_cancel, - ) - }); - let receipt = await_offline_writer(&cancel, signal.as_mut(), writer).await?; - Ok((receipt, inventory_captured_at, inventory_age)) - }; - let (runtime, (receipt, inventory_captured_at, inventory_age)) = run_offline_runtime(runtime, operation)?; - drop(runtime); - - println!("Bundle: {}", receipt.bundle_uid); - println!("Device: {}", receipt.device_name); - println!("Inventory: {} ({}s old)", inventory_captured_at, inventory_age.as_secs()); - println!("L0: {} entries, {} bytes", receipt.l0_count, receipt.l0_bytes); - println!("L1: {} entries, {} bytes", receipt.l1_count, receipt.l1_bytes); - println!( - "Redaction: {} ({})", - rustfs::connect::offline::redaction::REDACTION_VERSION, - rustfs::connect::offline::redaction::RULESET_HASH - ); - println!("Archive: {} bytes, sha256 {}", receipt.archive_size_bytes, receipt.archive_sha256); - println!("Output: {output_path}"); - println!("Upload: not performed"); - - Ok(()) -} - -async fn await_offline_writer( - cancel: &CancellationToken, - mut signal: Pin<&mut S>, - mut writer: tokio::task::JoinHandle>, -) -> Result -where - S: Future> + ?Sized, -{ - tokio::select! { - biased; - signal = signal.as_mut() => { - cancel.cancel(); - let writer = finish_offline_writer(writer.await); - match writer { - Ok(receipt) => Ok(receipt), - Err(error) => { - signal?; - Err(error) - } - } - } - result = &mut writer => finish_offline_writer(result), - } -} - -fn finish_offline_writer( - result: Result, tokio::task::JoinError>, -) -> Result { - result - .map_err(|error| format!("offline bundle writer task failed: {error}"))? - .map_err(|error| error.to_string()) -} - -fn run_offline_runtime( - runtime: tokio::runtime::Runtime, - operation: impl Future>, -) -> Result<(tokio::runtime::Runtime, T), String> { - match runtime.block_on(operation) { - Ok(value) => Ok((runtime, value)), - Err(error) => { - runtime.shutdown_timeout(OFFLINE_RUNTIME_SHUTDOWN_TIMEOUT); - Err(error) - } - } -} - -fn enroll(arguments: &[String]) -> Result<(), String> { - let mut challenge_path: Option = None; - let mut output_path: Option = None; - let mut key_directory: Option = None; - - let mut index = 0; - while index < arguments.len() { - let flag = arguments[index].as_str(); - let take_value = |name: &str| -> Result { - arguments - .get(index + 1) - .cloned() - .ok_or_else(|| format!("`{name}` needs a value\n\n{USAGE}")) - }; - - match flag { - "--challenge" => challenge_path = Some(take_value("--challenge")?), - "--output" => output_path = Some(take_value("--output")?), - "--key-dir" => key_directory = Some(take_value("--key-dir")?), - "-h" | "--help" => { - println!("{USAGE}"); - return Ok(()); - } - other => return Err(format!("unknown option `{other}`\n\n{USAGE}")), - } - - index += 2; - } - - let challenge_path = challenge_path.ok_or_else(|| format!("`--challenge` is required\n\n{USAGE}"))?; - let output_path = output_path.ok_or_else(|| format!("`--output` is required\n\n{USAGE}"))?; - let key_directory = key_directory.unwrap_or_else(|| ".".to_string()); - - let challenge = read_challenge(&challenge_path)?; - - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| "the system clock is before the Unix epoch".to_string())? - .as_secs() as i64; - #[cfg(feature = "offline-enrollment-e2e-root")] - let now = enrollment_evaluation_time(now)?; - - #[cfg(feature = "offline-enrollment-e2e-root")] - let verified = verify_enrollment_challenge(&challenge, now).map_err(|error| error.to_string())?; - #[cfg(not(feature = "offline-enrollment-e2e-root"))] - let verified = OfflineEnrollment::verify_challenge(&challenge, now).map_err(|error| error.to_string())?; - - // First use mints the key; a retry answers with the one already enrolled, - // because the operator may already be carrying a response naming it. - let key = OfflineKeyStore::new(&key_directory) - .load_or_create() - .map_err(|error| error.to_string())?; - - let mut device_nonce = [0u8; 32]; - getrandom(&mut device_nonce)?; - - let response = OfflineEnrollment::build_response(&verified, &key, &device_nonce, now).map_err(|error| error.to_string())?; - - write_response(Path::new(&output_path), &response)?; - - Ok(()) -} - -#[cfg(feature = "offline-enrollment-e2e-root")] -fn enrollment_evaluation_time(system_now: i64) -> Result { - // The public fixture gate needs a compile-time clock; real E2E builds omit it. - if env!("CARGO_BIN_NAME") == "rustfs-cli-e2e" - && let Some(value) = option_env!("RUSTFS_E2E_OFFLINE_ENROLLMENT_FIXTURE_TIME") - { - return value - .parse() - .map_err(|_| "the compiled E2E enrollment fixture time is invalid".to_owned()); - } - - Ok(system_now) -} - -#[cfg(feature = "offline-enrollment-e2e-root")] -fn verify_enrollment_challenge( - challenge: &[u8], - now: i64, -) -> Result { - if env!("CARGO_BIN_NAME") == "rustfs-cli-e2e" { - return OfflineEnrollment::verify_e2e_challenge(challenge, now); - } - - OfflineEnrollment::verify_challenge(challenge, now) -} - -/// Reads the challenge from a file, or from stdin when the path is `-`. -/// -/// A challenge is not a secret — it is signed, public, and carried in by hand — -/// so accepting a path is safe. The response's key never arrives this way. -fn read_challenge(path: &str) -> Result, String> { - if path == "-" { - let mut buffer = Vec::new(); - std::io::stdin() - .read_to_end(&mut buffer) - .map_err(|error| format!("cannot read the challenge from stdin: {error}"))?; - return Ok(buffer); - } - - fs::read(path).map_err(|error| format!("cannot read the challenge at {path}: {error}")) -} - -/// Writes the response durably and atomically at mode 0600. -/// -/// Not the no-clobber publish `IdentityStore` performs for a key: an operator -/// who reruns an enrolment expects the response file to be replaced, whereas a -/// second key would strand the first. Same durability, deliberately different -/// publication rule. -fn write_response(path: &Path, response: &[u8]) -> Result<(), String> { - let parent = path.parent().filter(|parent| !parent.as_os_str().is_empty()); - let temporary: PathBuf = match parent { - Some(parent) => parent.join(format!(".{}.tmp", file_name(path))), - None => PathBuf::from(format!(".{}.tmp", file_name(path))), - }; - - let mut options = fs::OpenOptions::new(); - options.write(true).create(true).truncate(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - options.mode(RESPONSE_MODE); - } - - let write = (|| -> std::io::Result<()> { - let mut file = options.open(&temporary)?; - file.write_all(response)?; - - // The umask can only narrow the creation mode, so set the exact mode - // before the bytes become durable. - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - file.set_permissions(fs::Permissions::from_mode(RESPONSE_MODE))?; - } - - file.sync_all() - })(); - - if let Err(error) = write { - let _ = fs::remove_file(&temporary); - return Err(format!("cannot write the response to {}: {error}", path.display())); - } - - fs::rename(&temporary, path).map_err(|error| { - let _ = fs::remove_file(&temporary); - format!("cannot publish the response at {}: {error}", path.display()) - })?; - - if let Some(parent) = parent { - sync_directory(parent); - } - - Ok(()) -} - -fn file_name(path: &Path) -> String { - path.file_name() - .map(|name| name.to_string_lossy().into_owned()) - .unwrap_or_else(|| "response".to_string()) -} - -/// Fsync the directory so the renamed entry survives power loss. Directories -/// cannot be opened for syncing on Windows, where this is a no-op. -fn sync_directory(directory: &Path) { - #[cfg(unix)] - { - if let Ok(handle) = fs::File::open(directory) { - let _ = handle.sync_all(); - } - } - #[cfg(not(unix))] - let _ = directory; -} - -/// Fills `buffer` with operating-system randomness. -/// -/// The device nonce must be unpredictable: it is what stops a captured response -/// being replayed as a fresh one. Use the workspace rand 0.10 system RNG, -/// matching the rand_core version used by the upgraded crypto stack. -fn getrandom(buffer: &mut [u8]) -> Result<(), String> { - use rand::{TryRng as _, rngs::SysRng}; - - SysRng - .try_fill_bytes(buffer) - .map_err(|error| format!("the operating system random source failed: {error}")) -} - -#[cfg(test)] -mod tests { - use std::time::Instant; - - use super::*; - - #[cfg(feature = "offline-enrollment-e2e-root")] - #[test] - fn only_the_dedicated_e2e_target_selects_the_test_root() { - let challenge = - fs::read(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/offline-enrollment-e2e/challenge.json")) - .expect("read E2E enrollment challenge"); - let result = verify_enrollment_challenge(&challenge, 4_070_908_800); - - if env!("CARGO_BIN_NAME") == "rustfs-cli-e2e" { - result.expect("the dedicated E2E binary selects the test root"); - } else { - assert_eq!( - result - .expect_err("every production binary must retain the hosted root") - .reason(), - "ENROLLMENT_ROOT_UNKNOWN" - ); - } - } - - #[test] - fn connect_offline_bundle_cli_shutdown_is_bounded_with_a_stuck_blocking_collector() { - let runtime = tokio::runtime::Builder::new_current_thread().build().expect("test runtime"); - let operation = async { - let (started, observed) = tokio::sync::oneshot::channel(); - tokio::task::spawn_blocking(move || { - started.send(()).expect("report blocking task"); - std::thread::sleep(Duration::from_secs(5)); - }); - observed.await.expect("blocking task started"); - Err::<(), _>("collector cancelled".to_owned()) - }; - - let started_at = Instant::now(); - assert!(run_offline_runtime(runtime, operation).is_err()); - assert!(started_at.elapsed() < Duration::from_secs(1)); - } - - #[test] - fn connect_offline_bundle_cli_preserves_success_when_signal_and_committed_writer_are_ready() { - let runtime = tokio::runtime::Builder::new_current_thread().build().expect("test runtime"); - let cancel = CancellationToken::new(); - let temp = tempfile::tempdir().expect("output tempdir"); - let output = temp.path().join("bundle.zip"); - let writer_output = output.clone(); - let signal = std::future::ready(Ok(())); - tokio::pin!(signal); - let writer = runtime.spawn(async move { - fs::write(writer_output, b"published bundle").expect("publish test bundle"); - Ok(BundleReceipt { - bundle_uid: "0198f3a1-8000-7e50-8f61-4a5b6c7d8e94".to_owned(), - device_name: "organizations/0198f3a1-4c00-7a10-8b21-0c1d2e3f4a50/clusters/0198f3a1-5d00-7b20-9c31-1d2e3f4a5b61/clusterDevices/0198f3a1-6e00-7c30-ad41-2e3f4a5b6c72".to_owned(), - archive_size_bytes: 1, - archive_sha256: "00".repeat(32), - l0_count: 6, - l0_bytes: 1, - l1_count: 6, - l1_bytes: 1, - }) - }); - runtime.block_on(async { - while !writer.is_finished() { - tokio::task::yield_now().await; - } - }); - assert_eq!(fs::read(&output).expect("read committed output"), b"published bundle"); - - let result = runtime.block_on(await_offline_writer(&cancel, signal.as_mut(), writer)); - assert!(result.is_ok()); - assert!(cancel.is_cancelled()); - } + common::run() } diff --git a/rustfs/src/connect/offline/enrollment.rs b/rustfs/src/connect/offline/enrollment.rs index ef36ce4cd..350c4ac28 100644 --- a/rustfs/src/connect/offline/enrollment.rs +++ b/rustfs/src/connect/offline/enrollment.rs @@ -51,10 +51,9 @@ const HOSTED_ROOT: EnrollmentRoot = EnrollmentRoot { }; #[cfg(feature = "offline-enrollment-e2e-root")] -const E2E_ROOT: EnrollmentRoot = EnrollmentRoot { - key_id: env!("RUSTFS_E2E_OFFLINE_ENROLLMENT_ROOT_KEY_ID"), - public_key: env!("RUSTFS_E2E_OFFLINE_ENROLLMENT_ROOT_PUBLIC_KEY"), -}; +const E2E_ROOT_KEY_ID: Option<&str> = option_env!("RUSTFS_E2E_OFFLINE_ENROLLMENT_ROOT_KEY_ID"); +#[cfg(feature = "offline-enrollment-e2e-root")] +const E2E_ROOT_PUBLIC_KEY: Option<&str> = option_env!("RUSTFS_E2E_OFFLINE_ENROLLMENT_ROOT_PUBLIC_KEY"); #[derive(Clone, Copy)] struct EnrollmentRoot { @@ -348,7 +347,7 @@ impl OfflineEnrollment { #[cfg(feature = "offline-enrollment-e2e-root")] #[doc(hidden)] pub fn verify_e2e_challenge(document: &[u8], now_unix: i64) -> Result { - Self::verify_challenge_with_root(document, now_unix, E2E_ROOT) + Self::verify_challenge_with_root(document, now_unix, e2e_root()?) } fn verify_challenge_with_root( @@ -462,6 +461,18 @@ impl OfflineEnrollment { } } +#[cfg(feature = "offline-enrollment-e2e-root")] +fn e2e_root() -> Result { + let key_id = E2E_ROOT_KEY_ID + .filter(|value| !value.is_empty()) + .ok_or(EnrollmentError::EnrollmentRootUnknown)?; + let public_key = E2E_ROOT_PUBLIC_KEY + .filter(|value| !value.is_empty()) + .ok_or(EnrollmentError::EnrollmentRootUnknown)?; + + Ok(EnrollmentRoot { key_id, public_key }) +} + /// Walk the chain from the pinned root to the signing key, returning the key /// `connect_key_id` names once the chain vouches for it. fn verify_trust_chain(