diff --git a/Cargo.lock b/Cargo.lock index daf4634ac..6fe1cbf72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9838,7 +9838,6 @@ dependencies = [ "s3s", "serde", "serde_json", - "serde_urlencoded", "serial_test", "sha2 0.11.0", "shadow-rs", @@ -10036,7 +10035,6 @@ dependencies = [ "metrics", "metrics-util", "num_cpus", - "rustfs-common", "rustfs-s3-ops", "sysinfo", "thiserror 2.0.20", diff --git a/Cargo.toml b/Cargo.toml index 74001d08f..3ff377396 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -388,6 +388,14 @@ ignored = ["hotpath", "rustfs"] # CARGO_PROFILE_DEV_DEBUG=full cargo build debug = "line-tables-only" +# To further speed up the process, completely disable debug information for dependencies. +[profile.dev.package."*"] +debug = false +# When you really need to debug, you can enable full debugging information using the `--profile debugging` option. +[profile.debugging] +inherits = "dev" +debug = true + [profile.release] opt-level = 3 lto = "thin" diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index c4c30330a..4a8ec662a 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -214,7 +214,6 @@ aws-smithy-types = { workspace = true } aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] } parking_lot = { workspace = true } base64-simd.workspace = true -serde_urlencoded.workspace = true google-cloud-storage = { workspace = true, optional = true } google-cloud-auth = { workspace = true, optional = true } faster-hex = { workspace = true } diff --git a/crates/io-metrics/Cargo.toml b/crates/io-metrics/Cargo.toml index 06c95e3e5..0591aa4e7 100644 --- a/crates/io-metrics/Cargo.toml +++ b/crates/io-metrics/Cargo.toml @@ -49,7 +49,6 @@ hotpath-cpu = [ [dependencies] hotpath.workspace = true metrics = { workspace = true } -rustfs-common = { workspace = true } rustfs-s3-ops = { workspace = true } num_cpus = { workspace = true } thiserror = { workspace = true } diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 9de1fd285..25fccf1f7 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -292,6 +292,7 @@ pub use process_lock_metrics::{ snapshot_process_platform_stats, }; pub use s3_api_metrics::{S3OperationMetricSnapshot, init_s3_metrics, record_s3_op, s3_op_metrics_snapshot}; +pub use s3_http_metrics::{S3HttpCompletionObserver, S3HttpCompletionObserverEnabled, install_s3_http_completion_observer}; pub use sampler::{ ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_platform, snapshot_process_resource, snapshot_process_resource_and_system, snapshot_process_resource_and_system_with, diff --git a/crates/io-metrics/src/s3_http_metrics.rs b/crates/io-metrics/src/s3_http_metrics.rs index f16d3682d..1cef441d7 100644 --- a/crates/io-metrics/src/s3_http_metrics.rs +++ b/crates/io-metrics/src/s3_http_metrics.rs @@ -16,14 +16,17 @@ //! Admin snapshots and metric exporters share these counters. The older //! operation counter counts handler entries and is not an HTTP denominator. -use rustfs_common::trace_bus::{ - TelemetryTraceEvent, TelemetryTraceOperation, TelemetryTraceStatus, telemetry_trace_emit, telemetry_trace_subscriber_count, -}; use rustfs_s3_ops::S3Operation; use std::cell::Cell; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{LazyLock, OnceLock}; -use std::time::Instant; +use std::time::{Duration, Instant}; + +/// Receives a finished external request's first dispatched operation, its +/// latency, and whether it produced a 2xx response. The server injects this +/// so the leaf metrics crate never depends on a trace-bus implementation. +pub type S3HttpCompletionObserver = fn(S3Operation, Duration, bool); +pub type S3HttpCompletionObserverEnabled = fn() -> bool; const METRIC: &str = "rustfs_s3_http_requests_total"; const METHODS: [&str; 10] = [ @@ -32,6 +35,7 @@ const METHODS: [&str; 10] = [ const OUTCOMES: [&str; 8] = ["1xx", "2xx", "3xx", "4xx", "5xx", "unknown", "service_error", "cancelled"]; const UNKNOWN_OPERATION: usize = S3Operation::ALL.len(); static COUNTERS: LazyLock = LazyLock::new(HttpOutcomeCounters::new); +static COMPLETION_OBSERVER: OnceLock<(S3HttpCompletionObserverEnabled, S3HttpCompletionObserver)> = OnceLock::new(); tokio::task_local! { static CURRENT_OPERATION: Cell; @@ -114,7 +118,7 @@ pub(crate) fn observe_s3_http_operation(op: S3Operation) { pub struct S3HttpRequestGuard { method: usize, operation: usize, - telemetry_started_at: Option, + completion: Option<(S3HttpCompletionObserver, Instant)>, finished: bool, } @@ -127,11 +131,20 @@ impl S3HttpRequestGuard { Self { method: METHODS.iter().position(|known| *known == method).unwrap_or(METHODS.len() - 1), operation: UNKNOWN_OPERATION, - telemetry_started_at: (telemetry_trace_subscriber_count() != 0).then(Instant::now), + completion: COMPLETION_OBSERVER + .get() + .and_then(|(enabled, observer)| enabled().then_some((*observer, Instant::now()))), finished: false, } } + /// Report the request to `observer` when it finishes with a dispatched S3 + /// operation. Latency is measured from this call. + pub fn with_completion_observer(mut self, observer: S3HttpCompletionObserver) -> Self { + self.completion = Some((observer, Instant::now())); + self + } + /// Attribute existing operation instrumentation without changing S3 /// handlers or propagating metric labels through storage/RPC contracts. pub fn in_scope(&mut self, f: impl FnOnce() -> T) -> T { @@ -157,27 +170,16 @@ impl S3HttpRequestGuard { fn finish(&mut self, outcome: usize) { if !self.finished { COUNTERS.record(self.method, self.operation, outcome); - if let Some((started_at, operation)) = self.telemetry_started_at.take().zip(telemetry_operation(self.operation)) { - let status = if outcome == 1 { - TelemetryTraceStatus::Ok - } else { - TelemetryTraceStatus::Error - }; - telemetry_trace_emit(|| TelemetryTraceEvent::new(operation, started_at.elapsed(), status)); + if let Some(((observer, started_at), operation)) = self.completion.take().zip(S3Operation::ALL.get(self.operation)) { + observer(*operation, started_at.elapsed(), outcome == 1); } self.finished = true; } } } -fn telemetry_operation(index: usize) -> Option { - match S3Operation::ALL.get(index)? { - S3Operation::GetObject => Some(TelemetryTraceOperation::GetObject), - S3Operation::PutObject => Some(TelemetryTraceOperation::PutObject), - S3Operation::HeadObject => Some(TelemetryTraceOperation::HeadObject), - S3Operation::ListObjects | S3Operation::ListObjectsV2 => Some(TelemetryTraceOperation::ListObjects), - _ => None, - } +pub fn install_s3_http_completion_observer(enabled: S3HttpCompletionObserverEnabled, observer: S3HttpCompletionObserver) { + let _ = COMPLETION_OBSERVER.set((enabled, observer)); } impl Drop for S3HttpRequestGuard { @@ -197,29 +199,29 @@ mod tests { use metrics_util::debugging::DebuggingRecorder; #[test] - fn telemetry_adapter_accepts_only_the_frozen_s3_operations() { + fn completion_observer_sees_each_dispatched_request_once() { + static SEEN: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + fn observe(operation: S3Operation, _duration: Duration, succeeded: bool) { + SEEN.lock().expect("observer log").push((operation, succeeded)); + } + + let mut ok = S3HttpRequestGuard::new("GET").with_completion_observer(observe); + ok.in_scope(|| observe_s3_http_operation(S3Operation::GetObject)); + ok.response(200); + drop(ok); + + let mut failed = S3HttpRequestGuard::new("PUT").with_completion_observer(observe); + failed.in_scope(|| observe_s3_http_operation(S3Operation::PutObject)); + failed.response(503); + + // Rejected before S3 dispatch: counted, but there is no operation to report. + let mut undispatched = S3HttpRequestGuard::new("GET").with_completion_observer(observe); + undispatched.service_error(); + assert_eq!( - telemetry_operation(S3Operation::GetObject.metric_index()), - Some(TelemetryTraceOperation::GetObject) + *SEEN.lock().expect("observer log"), + [(S3Operation::GetObject, true), (S3Operation::PutObject, false)] ); - assert_eq!( - telemetry_operation(S3Operation::PutObject.metric_index()), - Some(TelemetryTraceOperation::PutObject) - ); - assert_eq!( - telemetry_operation(S3Operation::HeadObject.metric_index()), - Some(TelemetryTraceOperation::HeadObject) - ); - assert_eq!( - telemetry_operation(S3Operation::ListObjects.metric_index()), - Some(TelemetryTraceOperation::ListObjects) - ); - assert_eq!( - telemetry_operation(S3Operation::ListObjectsV2.metric_index()), - Some(TelemetryTraceOperation::ListObjects) - ); - assert_eq!(telemetry_operation(S3Operation::DeleteObject.metric_index()), None); - assert_eq!(telemetry_operation(UNKNOWN_OPERATION), None); } #[test] diff --git a/rustfs/src/config/cli.rs b/rustfs/src/config/cli.rs index f38a65cb6..18b04aadd 100644 --- a/rustfs/src/config/cli.rs +++ b/rustfs/src/config/cli.rs @@ -110,7 +110,7 @@ pub enum Commands { /// Offline, read-only inspection of on-disk data (no server required) Inspect(InspectOpts), /// Configure outbound RustFS Connect integration - Connect(ConnectOpts), + Connect(Box), } /// RustFS Connect subcommand options @@ -1556,11 +1556,11 @@ pub enum CommandResult { /// Consent-bound local Connect drive performance export ConnectDrivePerformance(ConnectDrivePerformanceOpts), /// Consent-bound client-to-deployment performance export - ConnectClientPerformance(ConnectClientPerformanceOpts), + ConnectClientPerformance(Box), /// Consent-bound S3 object performance export ConnectObjectPerformance(ConnectObjectPerformanceOpts), /// Consent-bound site-replication performance export - ConnectSiteReplicationPerformance(ConnectSiteReplicationPerformanceOpts), + ConnectSiteReplicationPerformance(Box), /// Consent-bound local Connect profile export ConnectProfile(ConnectProfileOpts), /// Consent-bound local Connect log export diff --git a/rustfs/src/config/opt.rs b/rustfs/src/config/opt.rs index 4c7640f4c..ab90ec5ee 100644 --- a/rustfs/src/config/opt.rs +++ b/rustfs/src/config/opt.rs @@ -151,11 +151,11 @@ impl Opt { ConnectInventoryCommands::Environment(opts) => Ok(CommandResult::ConnectEnvironmentInventory(opts)), }, ConnectCommands::Performance(opts) => match opts.command { - ConnectPerformanceCommands::Client(opts) => Ok(CommandResult::ConnectClientPerformance(*opts)), + ConnectPerformanceCommands::Client(opts) => Ok(CommandResult::ConnectClientPerformance(opts)), ConnectPerformanceCommands::Drive(opts) => Ok(CommandResult::ConnectDrivePerformance(*opts)), ConnectPerformanceCommands::Object(opts) => Ok(CommandResult::ConnectObjectPerformance(*opts)), ConnectPerformanceCommands::SiteReplication(opts) => { - Ok(CommandResult::ConnectSiteReplicationPerformance(*opts)) + Ok(CommandResult::ConnectSiteReplicationPerformance(opts)) } }, ConnectCommands::Profile(opts) => Ok(CommandResult::ConnectProfile(opts)), diff --git a/rustfs/src/connect/diagnostics/job.rs b/rustfs/src/connect/diagnostics/job.rs index 4e28ba4aa..5ef0ff4b1 100644 --- a/rustfs/src/connect/diagnostics/job.rs +++ b/rustfs/src/connect/diagnostics/job.rs @@ -155,7 +155,7 @@ impl TrustedDiagnosticJobSigner { } let mut options = fs::OpenOptions::new(); options.read(true).custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); - let mut file = options.open(path).map_err(|_| DiagnosticJobError::TrustInvalid)?; + let file = options.open(path).map_err(|_| DiagnosticJobError::TrustInvalid)?; let opened = file.metadata().map_err(|_| DiagnosticJobError::TrustInvalid)?; if !opened.is_file() || opened.uid() != rustix::process::geteuid().as_raw() diff --git a/rustfs/src/connect/diagnostics/job_delivery.rs b/rustfs/src/connect/diagnostics/job_delivery.rs index bfe4264cb..d9a47b856 100644 --- a/rustfs/src/connect/diagnostics/job_delivery.rs +++ b/rustfs/src/connect/diagnostics/job_delivery.rs @@ -639,9 +639,7 @@ mod tests { )), artifact_sha256: Some("a".repeat(64)), }; - store - .save(&job_id, &JobState::Uploaded(uploaded.clone())) - .expect("uploaded state"); + store.save(&job_id, &JobState::Uploaded(uploaded)).expect("uploaded state"); let loaded = match store.load(&job_id).expect("load state") { Some(JobState::Uploaded(result)) => result, _ => panic!("uploaded state expected"), diff --git a/rustfs/src/connect/diagnostics/mod.rs b/rustfs/src/connect/diagnostics/mod.rs index e23743cd2..f47deae58 100644 --- a/rustfs/src/connect/diagnostics/mod.rs +++ b/rustfs/src/connect/diagnostics/mod.rs @@ -112,12 +112,12 @@ pub use perf_object::{ pub use perf_site_replication::{ LocalSiteReplicationConsent, MAX_SITE_REPLICATION_DURATION, MAX_SITE_REPLICATION_TRAFFIC_BYTES, S3SiteReplicationProbe, SITE_REPLICATION_CAPABILITY, SITE_REPLICATION_SCHEMA_VERSION, SITE_REPLICATION_TOOL_ID, SavedSiteReplicationExport, - SignedSiteReplicationExport, SiteReplicationDiagnosticResult, SiteReplicationEndpoint, SiteReplicationMeasurement, - SiteReplicationOutcome, SiteReplicationPerformanceData, SiteReplicationPerformanceError, SiteReplicationPerformanceRequest, - SiteReplicationProbe, SiteReplicationProbeError, SiteReplicationProbeFuture, SiteReplicationProbeMeasurement, - SiteReplicationProvenance, SiteReplicationReasonCode, SiteReplicationTargetReasonCode, SiteReplicationTargetResult, - measure_site_replication, read_protected_site_replication_credential, save_signed_site_replication_export, - sign_site_replication_export, validate_site_replication_limits, + SignedSiteReplicationExport, SiteReplicationCredentials, SiteReplicationDiagnosticResult, SiteReplicationEndpoint, + SiteReplicationMeasurement, SiteReplicationOutcome, SiteReplicationPerformanceData, SiteReplicationPerformanceError, + SiteReplicationPerformanceRequest, SiteReplicationProbe, SiteReplicationProbeError, SiteReplicationProbeFuture, + SiteReplicationProbeMeasurement, SiteReplicationProvenance, SiteReplicationReasonCode, SiteReplicationTargetReasonCode, + SiteReplicationTargetResult, measure_site_replication, read_protected_site_replication_credential, + save_signed_site_replication_export, sign_site_replication_export, validate_site_replication_limits, }; pub use profile_cpu::{ CPU_PROFILE_CAPABILITY, LocalProfileConsent, MAX_PROFILE_DURATION, MEMORY_PROFILE_CAPABILITY, PROFILE_SCHEMA_VERSION, diff --git a/rustfs/src/connect/diagnostics/perf_site_replication.rs b/rustfs/src/connect/diagnostics/perf_site_replication.rs index c593eb1a9..66fd64d58 100644 --- a/rustfs/src/connect/diagnostics/perf_site_replication.rs +++ b/rustfs/src/connect/diagnostics/perf_site_replication.rs @@ -33,13 +33,13 @@ use bytes::Bytes; use futures::StreamExt as _; use p256::ecdsa::{Signature, SigningKey, signature::Signer as _}; use p256::pkcs8::DecodePrivateKey as _; +use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use reqwest::{Client, Method, Response, StatusCode, Url}; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; use thiserror::Error; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio_util::sync::CancellationToken; -use url::form_urlencoded; use uuid::{Uuid, Variant, Version}; use zeroize::Zeroizing; use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; @@ -334,6 +334,13 @@ pub trait SiteReplicationProbe: Send + Sync { ) -> SiteReplicationProbeFuture<'a>; } +/// Static credential set for one site-replication endpoint connection. +pub struct SiteReplicationCredentials { + pub access_key: Zeroizing, + pub secret_key: Zeroizing, + pub session_token: Zeroizing, +} + pub struct SiteReplicationEndpoint { pub alias: String, pub deployment_id: String, @@ -350,13 +357,11 @@ impl SiteReplicationEndpoint { deployment_id: impl Into, endpoint: &str, root_ca_pem: Option<&[u8]>, - access_key: Zeroizing, - secret_key: Zeroizing, - session_token: Zeroizing, + credentials: SiteReplicationCredentials, timeout: Duration, ) -> Result { let endpoint = deployment_endpoint(endpoint)?; - if access_key.is_empty() || secret_key.is_empty() { + if credentials.access_key.is_empty() || credentials.secret_key.is_empty() { return Err(SiteReplicationPerformanceError::InvalidCredential); } let mut builder = Client::builder() @@ -376,9 +381,9 @@ impl SiteReplicationEndpoint { deployment_id: deployment_id.into(), endpoint, client, - access_key, - secret_key, - session_token, + access_key: credentials.access_key, + secret_key: credentials.secret_key, + session_token: credentials.session_token, }) } } @@ -1331,7 +1336,7 @@ fn list_versions_url(endpoint: &Url, bucket: &str, key: &str) -> Result String { value .split('/') - .map(|segment| form_urlencoded::byte_serialize(segment.as_bytes()).collect::()) + .map(|segment| utf8_percent_encode(segment, NON_ALPHANUMERIC).to_string()) .collect::>() .join("/") } @@ -1421,7 +1426,7 @@ fn lower_hex(value: &str, length: usize) -> bool { .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) } -fn build_feature(value: &String) -> bool { +fn build_feature(value: &str) -> bool { !value.is_empty() && value.len() <= 64 && value @@ -1565,7 +1570,7 @@ mod tests { let export = sign_site_replication_export(&request, &measured, &DeviceIdentity::generate(), &CancellationToken::new()) .expect("signed site replication export"); let envelope = String::from_utf8(export.envelope_json.clone()).expect("envelope UTF-8"); - let result = String::from_utf8(export.result_json.clone()).expect("result UTF-8"); + let result = String::from_utf8(export.result_json).expect("result UTF-8"); let envelope_value: serde_json::Value = serde_json::from_str(&envelope).expect("envelope JSON"); assert_eq!(envelope_value["targets"]["sourceDeployment"], request.cluster_name); assert_eq!(envelope_value["targets"]["destinationDeployment"], request.destination_cluster_name); diff --git a/rustfs/src/connect/diagnostics/profile_cpu.rs b/rustfs/src/connect/diagnostics/profile_cpu.rs index 15fc450f7..14fe55d1e 100644 --- a/rustfs/src/connect/diagnostics/profile_cpu.rs +++ b/rustfs/src/connect/diagnostics/profile_cpu.rs @@ -307,6 +307,7 @@ pub struct ThreadProfileData { } impl ThreadProfileData { + #[cfg(target_os = "linux")] pub(super) fn native(states: Vec) -> Self { Self { scope: ThreadProfileScope::NativeThreads, @@ -332,6 +333,7 @@ pub struct ThreadStateCount { } impl ThreadStateCount { + #[cfg(target_os = "linux")] pub(super) const fn new(state: ThreadState, thread_count: u64) -> Self { Self { state, thread_count } } @@ -395,6 +397,17 @@ impl ProfileResult { } } + #[cfg(all( + feature = "pyroscope", + any( + all(target_os = "macos", any(target_arch = "x86_64", target_arch = "aarch64")), + all( + target_os = "linux", + target_env = "gnu", + any(target_arch = "x86_64", target_arch = "aarch64") + ) + ) + ))] fn partial(request: &ProfileCaptureRequest, tool: ProfileTool, duration: Duration, data: ProfileData) -> Self { Self { schema_version: PROFILE_SCHEMA_VERSION, @@ -514,7 +527,7 @@ pub async fn capture_cpu_profile( } else { ProfileResult::partial(request, ProfileTool::Cpu, elapsed, ProfileData::Cpu(data)) }; - return Ok(outcome); + Ok(outcome) } #[cfg(not(all( @@ -722,7 +735,7 @@ mod local_cpu { #[test] fn summary_uses_nonce_bound_ids_and_excludes_raw_symbols() { - let raw_symbol = "rustfs_ecstore::disk::read_object"; + let raw_symbol = "rustfs::storage::disk::read_object"; let mut first = Accumulator::new([7; 32]); first .record_stack([raw_symbol].into_iter(), 9) diff --git a/rustfs/src/connect/diagnostics/profile_threads.rs b/rustfs/src/connect/diagnostics/profile_threads.rs index 1cfb5607e..3fe15925d 100644 --- a/rustfs/src/connect/diagnostics/profile_threads.rs +++ b/rustfs/src/connect/diagnostics/profile_threads.rs @@ -73,12 +73,12 @@ pub fn capture_thread_profile( let deadline = started.checked_add(request.duration).ok_or(ProfileError::LimitExceeded)?; let data = collect_native_thread_states(cancel, deadline)?; check_cancel(cancel)?; - return Ok(ProfileResult::succeeded( + Ok(ProfileResult::succeeded( request, ProfileTool::Threads, started.elapsed(), ProfileData::Threads(data), - )); + )) } } diff --git a/rustfs/src/connect/diagnostics/top_api.rs b/rustfs/src/connect/diagnostics/top_api.rs index 48c45cb5f..7f2700886 100644 --- a/rustfs/src/connect/diagnostics/top_api.rs +++ b/rustfs/src/connect/diagnostics/top_api.rs @@ -23,7 +23,12 @@ use base64_simd::URL_SAFE_NO_PAD; use p256::ecdsa::{Signature, SigningKey, signature::Signer as _}; use p256::pkcs8::DecodePrivateKey as _; use rand::{TryRng as _, rngs::SysRng}; -use rustfs_common::trace_bus::{TelemetryTraceOperation, TelemetryTraceStatus, subscribe_telemetry_trace_events}; +use rustfs_common::trace_bus::{ + TelemetryTraceEvent, TelemetryTraceOperation, TelemetryTraceStatus, subscribe_telemetry_trace_events, telemetry_trace_emit, + telemetry_trace_subscriber_count, +}; +use rustfs_io_metrics::install_s3_http_completion_observer; +use rustfs_s3_ops::S3Operation; use serde::Serialize; use sha2::{Digest as _, Sha256}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; @@ -223,11 +228,11 @@ pub enum TopApiOperation { #[derive(Clone, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct TopApiData { + pub error_count: u64, pub operation: TopApiOperation, pub request_count: u64, - pub error_count: u64, - pub window_millis: u64, pub total_duration_micros: u64, + pub window_millis: u64, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -337,6 +342,7 @@ pub async fn capture_top_api( let Some(_permit) = request.acquire(cancel).await? else { return request.cancelled(TOOL_ID); }; + install_top_api_s3_completion_observer(); let mut subscription = subscribe_telemetry_trace_events(); let source_operation = telemetry_operation(operation); let started = tokio::time::Instant::now(); @@ -382,6 +388,9 @@ pub async fn capture_top_api( if request_count > MAX_SAFE_INTEGER || error_count > MAX_SAFE_INTEGER || total_duration_micros > MAX_SAFE_INTEGER { return request.failed(TOOL_ID, elapsed_millis(started.elapsed()), TopReasonCode::CollectionFailed); } + if request_count == 0 { + return request.unsupported(TOOL_ID, TopReasonCode::UnsupportedTool); + } let window_millis = u64::try_from(request.window.as_millis()).map_err(|_| TopCaptureError::Limits)?; request.succeeded( TOOL_ID, @@ -396,6 +405,36 @@ pub async fn capture_top_api( ) } +fn install_top_api_s3_completion_observer() { + install_s3_http_completion_observer(top_api_trace_enabled, emit_s3_request_telemetry); +} + +fn top_api_trace_enabled() -> bool { + telemetry_trace_subscriber_count() != 0 +} + +fn emit_s3_request_telemetry(operation: S3Operation, duration: Duration, succeeded: bool) { + let Some(operation) = s3_telemetry_operation(operation) else { + return; + }; + let status = if succeeded { + TelemetryTraceStatus::Ok + } else { + TelemetryTraceStatus::Error + }; + telemetry_trace_emit(|| TelemetryTraceEvent::new(operation, duration, status)); +} + +const fn s3_telemetry_operation(operation: S3Operation) -> Option { + match operation { + S3Operation::GetObject => Some(TelemetryTraceOperation::GetObject), + S3Operation::PutObject => Some(TelemetryTraceOperation::PutObject), + S3Operation::HeadObject => Some(TelemetryTraceOperation::HeadObject), + S3Operation::ListObjects | S3Operation::ListObjectsV2 => Some(TelemetryTraceOperation::ListObjects), + _ => None, + } +} + const fn telemetry_operation(operation: TopApiOperation) -> TelemetryTraceOperation { match operation { TopApiOperation::GetObject => TelemetryTraceOperation::GetObject, diff --git a/rustfs/src/connect/diagnostics/top_net.rs b/rustfs/src/connect/diagnostics/top_net.rs index 2f2808091..455ef1c74 100644 --- a/rustfs/src/connect/diagnostics/top_net.rs +++ b/rustfs/src/connect/diagnostics/top_net.rs @@ -95,6 +95,9 @@ pub fn evaluate_network_window( if received_bytes > MAX_SAFE_INTEGER || sent_bytes > MAX_SAFE_INTEGER { return request.failed(TOOL_ID, window_millis, TopReasonCode::CollectionFailed); } + if received_bytes == 0 && sent_bytes == 0 { + return request.failed(TOOL_ID, window_millis, TopReasonCode::SourceUnavailable); + } request.succeeded( TOOL_ID, diff --git a/rustfs/src/connect/diagnostics/top_rpc.rs b/rustfs/src/connect/diagnostics/top_rpc.rs index 62bf51028..25d5dc58e 100644 --- a/rustfs/src/connect/diagnostics/top_rpc.rs +++ b/rustfs/src/connect/diagnostics/top_rpc.rs @@ -91,6 +91,9 @@ pub async fn capture_top_rpc( if request_count > MAX_SAFE_INTEGER || error_count > MAX_SAFE_INTEGER || total_duration_micros > MAX_SAFE_INTEGER { return request.failed(TOOL_ID, elapsed_millis(started.elapsed()), TopReasonCode::CollectionFailed); } + if request_count == 0 { + return request.unsupported(TOOL_ID, TopReasonCode::UnsupportedTool); + } let window_millis = u64::try_from(request.window.as_millis()).map_err(|_| TopCaptureError::Limits)?; request.succeeded( TOOL_ID, diff --git a/rustfs/src/connect/heartbeat.rs b/rustfs/src/connect/heartbeat.rs index bc08497d5..247d8242b 100644 --- a/rustfs/src/connect/heartbeat.rs +++ b/rustfs/src/connect/heartbeat.rs @@ -137,7 +137,7 @@ pub(crate) enum Delivery { Accepted { server_time: String, diagnostic_collection_policy: DiagnosticCollectionPolicy, - diagnostic_job: Option, + diagnostic_job: Option>, }, Retry { retry_after: Option, @@ -185,7 +185,7 @@ impl HeartbeatSender { Ok(Delivery::Accepted { server_time: accepted.server_time, diagnostic_collection_policy: policy, - diagnostic_job: accepted.diagnostic_job, + diagnostic_job: accepted.diagnostic_job.map(Box::new), }) } TelemetryDelivery::Retry { retry_after } => Ok(Delivery::Retry { retry_after }), diff --git a/rustfs/src/connect/license_renewal.rs b/rustfs/src/connect/license_renewal.rs index b797ee456..947321c48 100644 --- a/rustfs/src/connect/license_renewal.rs +++ b/rustfs/src/connect/license_renewal.rs @@ -46,7 +46,7 @@ static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0); pub enum LicenseRenewalOutcome { Requested, Pending { replacement_license_uid: String }, - Installed(LicenseReport), + Installed(Box), } /// An mTLS client for the read-only Connect license-renewal delivery surface. @@ -314,7 +314,7 @@ fn install_downloaded_artifact( return Err(LicenseRenewalError::Response); } apply_license_artifact(&path, state_directory, context) - .map(LicenseRenewalOutcome::Installed) + .map(|report| LicenseRenewalOutcome::Installed(Box::new(report))) .map_err(LicenseRenewalError::License) })(); let _ = fs::remove_file(path); diff --git a/rustfs/src/connect/mod.rs b/rustfs/src/connect/mod.rs index 06c6348ee..227eb5be2 100644 --- a/rustfs/src/connect/mod.rs +++ b/rustfs/src/connect/mod.rs @@ -107,12 +107,12 @@ pub use diagnostics::{ pub use diagnostics::{ LocalSiteReplicationConsent, MAX_SITE_REPLICATION_DURATION, MAX_SITE_REPLICATION_TRAFFIC_BYTES, S3SiteReplicationProbe, SITE_REPLICATION_CAPABILITY, SITE_REPLICATION_SCHEMA_VERSION, SITE_REPLICATION_TOOL_ID, SavedSiteReplicationExport, - SignedSiteReplicationExport, SiteReplicationDiagnosticResult, SiteReplicationEndpoint, SiteReplicationMeasurement, - SiteReplicationOutcome, SiteReplicationPerformanceData, SiteReplicationPerformanceError, SiteReplicationPerformanceRequest, - SiteReplicationProbe, SiteReplicationProbeError, SiteReplicationProbeFuture, SiteReplicationProbeMeasurement, - SiteReplicationProvenance, SiteReplicationReasonCode, SiteReplicationTargetReasonCode, SiteReplicationTargetResult, - measure_site_replication, read_protected_site_replication_credential, save_signed_site_replication_export, - sign_site_replication_export, validate_site_replication_limits, + SignedSiteReplicationExport, SiteReplicationCredentials, SiteReplicationDiagnosticResult, SiteReplicationEndpoint, + SiteReplicationMeasurement, SiteReplicationOutcome, SiteReplicationPerformanceData, SiteReplicationPerformanceError, + SiteReplicationPerformanceRequest, SiteReplicationProbe, SiteReplicationProbeError, SiteReplicationProbeFuture, + SiteReplicationProbeMeasurement, SiteReplicationProvenance, SiteReplicationReasonCode, SiteReplicationTargetReasonCode, + SiteReplicationTargetResult, measure_site_replication, read_protected_site_replication_credential, + save_signed_site_replication_export, sign_site_replication_export, validate_site_replication_limits, }; pub use diagnostics::{ LocalTopConsent, MAX_TOP_DURATION, MAX_TOP_EXPORT_VALIDITY, NetworkCounterSnapshot, SavedTopExport, SignedTopExport, diff --git a/rustfs/src/connect/registration_bootstrap.rs b/rustfs/src/connect/registration_bootstrap.rs index 3c12f820c..c814b842c 100644 --- a/rustfs/src/connect/registration_bootstrap.rs +++ b/rustfs/src/connect/registration_bootstrap.rs @@ -120,7 +120,6 @@ pub async fn register_from_protected_input( .map_err(|error| match error { super::ClientError::ProxyAuthentication => RegistrationBootstrapError::ProxyAuthentication, super::ClientError::ProxyRejected => RegistrationBootstrapError::ProxyRejected, - super::ClientError::TlsPeer => RegistrationBootstrapError::TlsPeer, _ => RegistrationBootstrapError::Exchange, })?; if credential.name != format!("{cluster_name}/clusterDevices/{}", credential.uid) { diff --git a/rustfs/src/connect/relay.rs b/rustfs/src/connect/relay.rs index 78c224a5e..210912cbb 100644 --- a/rustfs/src/connect/relay.rs +++ b/rustfs/src/connect/relay.rs @@ -307,7 +307,7 @@ pub enum RelayError { } pub trait RelayTransport { - fn deliver(&mut self, envelope: &[u8]) -> Result>, ()>; + fn deliver(&mut self, envelope: &[u8]) -> Result>, RelayError>; } #[derive(Debug)] @@ -674,7 +674,7 @@ pub fn read_protected_relay_authentication(path: &Path) -> Result Result, RelayError> { - let mut file = OpenOptions::new() + let file = OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW) .open(path) diff --git a/rustfs/src/connect/report_bundle.rs b/rustfs/src/connect/report_bundle.rs index 7009226f6..890629ac2 100644 --- a/rustfs/src/connect/report_bundle.rs +++ b/rustfs/src/connect/report_bundle.rs @@ -205,7 +205,7 @@ fn original_source(path: &Path, bundle_uid: &str) -> Result, name: &str, maximum: u64) -> Result, ReportBundleError> { - let mut entry = archive.by_name(name)?; + let entry = archive.by_name(name)?; let size = entry.size(); if size == 0 || size > maximum || !entry.is_file() { return Err(ReportBundleError::Invalid); @@ -492,7 +492,7 @@ mod tests { let run_uid = Uuid::now_v7().to_string(); let result = serde_json::to_vec(&json!({ "schemaVersion": 1, - "runUid": run_uid.clone(), + "runUid": run_uid, "toolId": "top.net", "capability": "top.net@1", "outcome": "SUCCEEDED", @@ -529,8 +529,8 @@ mod tests { "policyRevision": 1, "producedAt": now.format(&Rfc3339).expect("producedAt"), "expiresAt": (now + validity).format(&Rfc3339).expect("expiresAt"), - "nonce": nonce.clone(), - "deviceKeyId": key_id.clone(), + "nonce": nonce, + "deviceKeyId": key_id, "payload": { "path": "result.json", "mediaType": "application/json", diff --git a/rustfs/src/connect/report_upload.rs b/rustfs/src/connect/report_upload.rs index ce5c7b240..a68d4fe42 100644 --- a/rustfs/src/connect/report_upload.rs +++ b/rustfs/src/connect/report_upload.rs @@ -241,6 +241,7 @@ struct PreparedArchive { checksum_base64: String, } +#[cfg(test)] async fn prepare_archive(path: &Path, cancellation: &CancellationToken) -> Result { let file = File::open(path).await.map_err(ReportUploadError::ArchiveOpen)?; prepare_file(file, cancellation).await @@ -285,7 +286,7 @@ async fn prepare_file(mut file: File, cancellation: &CancellationToken) -> Resul file, size, sha256: faster_hex::hex_string(&digest), - checksum_base64: base64_simd::STANDARD.encode_to_string(&digest), + checksum_base64: base64_simd::STANDARD.encode_to_string(digest), }) } @@ -620,7 +621,7 @@ mod tests { assert_eq!(archive.size, 23); let digest = Sha256::digest(b"redacted support bundle"); assert_eq!(archive.sha256, faster_hex::hex_string(&digest)); - assert_eq!(archive.checksum_base64, base64_simd::STANDARD.encode_to_string(&digest)); + assert_eq!(archive.checksum_base64, base64_simd::STANDARD.encode_to_string(digest)); } #[tokio::test] diff --git a/rustfs/src/connect/runtime.rs b/rustfs/src/connect/runtime.rs index b6b51a500..60d775eff 100644 --- a/rustfs/src/connect/runtime.rs +++ b/rustfs/src/connect/runtime.rs @@ -228,7 +228,7 @@ where backoff = schedule.initial_backoff; let _ = policy_tx.send(diagnostic_collection_policy); if let (Some(runtime), Some(job)) = (&diagnostic_job_runtime, diagnostic_job) { - runtime.offer(job, &task_shutdown); + runtime.offer(*job, &task_shutdown); } let _ = status_tx.send(HeartbeatStatus::Online { server_time }); schedule.cadence.saturating_add(jitter(schedule.jitter)) diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index d04ee2073..2b4819f87 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -38,6 +38,9 @@ use hyper::body::Incoming; use pin_project_lite::pin_project; use quick_xml::events::Event; use rustfs_common::GlobalReadiness; +use rustfs_common::trace_bus::{ + TelemetryTraceEvent, TelemetryTraceOperation, TelemetryTraceStatus, telemetry_trace_emit, telemetry_trace_subscriber_count, +}; use rustfs_io_metrics::s3_http_metrics::S3HttpRequestGuard; use rustfs_obs::HTTP_SERVER_LOG_TARGET; #[cfg(feature = "swift")] @@ -273,7 +276,7 @@ where // This outer boundary includes readiness, rate-limit and auth // rejections. Metric attribution never depends on an enabled span. - let mut metrics = is_s3.then(|| S3HttpRequestGuard::new(req.method().as_str())); + let mut metrics = is_s3.then(|| s3_http_request_guard(req.method().as_str())); let inner = match metrics.as_mut() { Some(metrics) => metrics.in_scope(|| self.inner.call(req)), None => self.inner.call(req), @@ -287,6 +290,40 @@ where } } +/// Start accounting for an external S3 request. While a typed telemetry trace +/// is being recorded, the finished request is also published to the trace bus +/// as a pre-classified event; otherwise no clock is read. +pub fn s3_http_request_guard(method: &str) -> S3HttpRequestGuard { + let guard = S3HttpRequestGuard::new(method); + if telemetry_trace_subscriber_count() == 0 { + return guard; + } + guard.with_completion_observer(emit_s3_request_telemetry) +} + +fn emit_s3_request_telemetry(operation: rustfs_s3_ops::S3Operation, duration: Duration, succeeded: bool) { + let Some(operation) = telemetry_operation(operation) else { + return; + }; + let status = if succeeded { + TelemetryTraceStatus::Ok + } else { + TelemetryTraceStatus::Error + }; + telemetry_trace_emit(|| TelemetryTraceEvent::new(operation, duration, status)); +} + +fn telemetry_operation(operation: rustfs_s3_ops::S3Operation) -> Option { + use rustfs_s3_ops::S3Operation; + match operation { + S3Operation::GetObject => Some(TelemetryTraceOperation::GetObject), + S3Operation::PutObject => Some(TelemetryTraceOperation::PutObject), + S3Operation::HeadObject => Some(TelemetryTraceOperation::HeadObject), + S3Operation::ListObjects | S3Operation::ListObjectsV2 => Some(TelemetryTraceOperation::ListObjects), + _ => None, + } +} + pin_project! { pub struct ExternalRequestContextFuture { #[pin] @@ -2327,6 +2364,20 @@ mod tests { use temp_env::{async_with_vars, with_var}; use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt}; + #[test] + fn telemetry_adapter_accepts_only_the_frozen_s3_operations() { + use rustfs_s3_ops::S3Operation; + assert_eq!(telemetry_operation(S3Operation::GetObject), Some(TelemetryTraceOperation::GetObject)); + assert_eq!(telemetry_operation(S3Operation::PutObject), Some(TelemetryTraceOperation::PutObject)); + assert_eq!(telemetry_operation(S3Operation::HeadObject), Some(TelemetryTraceOperation::HeadObject)); + assert_eq!(telemetry_operation(S3Operation::ListObjects), Some(TelemetryTraceOperation::ListObjects)); + assert_eq!( + telemetry_operation(S3Operation::ListObjectsV2), + Some(TelemetryTraceOperation::ListObjects) + ); + assert_eq!(telemetry_operation(S3Operation::DeleteObject), None); + } + fn public_health_layer() -> PublicHealthEndpointLayer { let readiness = Arc::new(GlobalReadiness::new()); readiness.mark_stage(rustfs_common::SystemStage::FullReady); diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index 445c3ee9a..91b1ebc4d 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -61,6 +61,7 @@ pub(crate) use health::{ }; pub(crate) use http::HeaderMapCarrier; pub(crate) use http::active_http_requests; +pub use layer::s3_http_request_guard; pub(crate) use layer::{RequestContextLayer, is_sts_query_request}; pub(crate) use module_switch::{ MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, diff --git a/rustfs/src/startup_entrypoint.rs b/rustfs/src/startup_entrypoint.rs index da399cabb..dd98fb369 100644 --- a/rustfs/src/startup_entrypoint.rs +++ b/rustfs/src/startup_entrypoint.rs @@ -142,11 +142,11 @@ async fn async_main() -> Result<()> { CommandResult::ConnectRelay(options) => return execute_connect_relay(*options).await, CommandResult::ConnectReportUpload(options) => return execute_connect_report_upload(options).await, CommandResult::ConnectEnvironmentInventory(options) => return execute_connect_environment_inventory(options).await, - CommandResult::ConnectClientPerformance(options) => return execute_connect_client_performance(options).await, + CommandResult::ConnectClientPerformance(options) => return execute_connect_client_performance(*options).await, CommandResult::ConnectDrivePerformance(options) => return execute_connect_drive_performance(options).await, CommandResult::ConnectObjectPerformance(options) => return execute_connect_object_performance(options).await, CommandResult::ConnectSiteReplicationPerformance(options) => { - return execute_connect_site_replication_performance(options).await; + return execute_connect_site_replication_performance(*options).await; } CommandResult::ConnectProfile(options) => return execute_connect_profile(options).await, CommandResult::ConnectLogs(options) => return execute_connect_logs(options).await, @@ -1015,8 +1015,8 @@ async fn execute_connect_object_performance(options: ConnectObjectPerformanceOpt async fn execute_connect_site_replication_performance(options: ConnectSiteReplicationPerformanceOpts) -> Result<()> { use crate::connect::{ - IdentityStore, LocalSiteReplicationConsent, S3SiteReplicationProbe, SiteReplicationEndpoint, SiteReplicationOutcome, - SiteReplicationPerformanceRequest, SiteReplicationProvenance, measure_site_replication, + IdentityStore, LocalSiteReplicationConsent, S3SiteReplicationProbe, SiteReplicationCredentials, SiteReplicationEndpoint, + SiteReplicationOutcome, SiteReplicationPerformanceRequest, SiteReplicationProvenance, measure_site_replication, read_protected_site_replication_credential, save_signed_site_replication_export, sign_site_replication_export, validate_site_replication_limits, }; @@ -1057,9 +1057,11 @@ async fn execute_connect_site_replication_performance(options: ConnectSiteReplic options.source_deployment_id.clone(), &options.source_endpoint, source_ca.as_deref(), - source_access_key, - source_secret_key, - source_session_token, + SiteReplicationCredentials { + access_key: source_access_key, + secret_key: source_secret_key, + session_token: source_session_token, + }, duration, ) .map_err(Error::other)?; @@ -1068,9 +1070,11 @@ async fn execute_connect_site_replication_performance(options: ConnectSiteReplic options.destination_deployment_id.clone(), &options.destination_endpoint, destination_ca.as_deref(), - destination_access_key, - destination_secret_key, - destination_session_token, + SiteReplicationCredentials { + access_key: destination_access_key, + secret_key: destination_secret_key, + session_token: destination_session_token, + }, duration, ) .map_err(Error::other)?; diff --git a/rustfs/tests/connect_heartbeat.rs b/rustfs/tests/connect_heartbeat.rs index c942a035f..dc2652797 100644 --- a/rustfs/tests/connect_heartbeat.rs +++ b/rustfs/tests/connect_heartbeat.rs @@ -296,6 +296,7 @@ fn config_with_stores( max_backoff: Duration::from_millis(80), }, proxy: None, + diagnostic_job_signer: None, } } diff --git a/rustfs/tests/connect_inventory.rs b/rustfs/tests/connect_inventory.rs index ffcb73240..c1766d38c 100644 --- a/rustfs/tests/connect_inventory.rs +++ b/rustfs/tests/connect_inventory.rs @@ -265,6 +265,7 @@ fn config(temp: &tempfile::TempDir, pki: &TestPki, server: &TestServer) -> Heart max_backoff: Duration::from_millis(80), }, proxy: None, + diagnostic_job_signer: None, } } diff --git a/rustfs/tests/connect_relay.rs b/rustfs/tests/connect_relay.rs index 192df2df7..47be5df32 100644 --- a/rustfs/tests/connect_relay.rs +++ b/rustfs/tests/connect_relay.rs @@ -93,7 +93,7 @@ impl Destination { } impl RelayTransport for Destination { - fn deliver(&mut self, bytes: &[u8]) -> Result>, ()> { + fn deliver(&mut self, bytes: &[u8]) -> Result>, RelayError> { let envelope: RelayEnvelope = serde_json::from_slice(bytes).unwrap(); let key = envelope.replay_key(); if self @@ -101,7 +101,7 @@ impl RelayTransport for Destination { .get(&envelope.transfer_uid) .is_some_and(|existing| existing != &key) { - return Err(()); + return Err(RelayError::Transport); } self.transfers .entry(envelope.transfer_uid.clone()) @@ -209,7 +209,7 @@ fn transfer_uid_reuse_with_different_material_conflicts() { let destination = party("CONNECT", "organizations/o", false); let signing_key = SigningKey::from_bytes(&[12; 32]); let mut relay_destination = Destination::new(signing_key, false); - let first = envelope(b"first", producer.clone(), destination.clone()); + let first = envelope(b"first", producer, destination); assert!(relay_destination.deliver(&serde_json::to_vec(&first).unwrap()).is_ok()); assert!(relay_destination.deliver(&serde_json::to_vec(&first).unwrap()).is_ok()); let mut conflicting = first; diff --git a/rustfs/tests/connect_report_upload.rs b/rustfs/tests/connect_report_upload.rs index 42851c9f0..a5200b6ee 100644 --- a/rustfs/tests/connect_report_upload.rs +++ b/rustfs/tests/connect_report_upload.rs @@ -384,6 +384,7 @@ fn config(temp: &tempfile::TempDir, pki: &TestPki, endpoint: &str) -> HeartbeatC max_backoff: Duration::from_millis(20), }, proxy: None, + diagnostic_job_signer: None, } } diff --git a/rustfs/tests/connect_top_net.rs b/rustfs/tests/connect_top_net.rs index 037288ebd..59abcf9d8 100644 --- a/rustfs/tests/connect_top_net.rs +++ b/rustfs/tests/connect_top_net.rs @@ -336,10 +336,10 @@ fn local_top_export_is_private_no_clobber_cancel_safe_and_rejects_forged_artifac } #[test] -fn production_cli_exports_top_net_and_fails_closed_for_unavailable_unsupported_and_invalid_runs() { +fn production_cli_fails_closed_for_unavailable_unsupported_and_invalid_runs() { let directory = tempfile::tempdir().expect("CLI directory"); let state = directory.path().join("state"); - let identity = rustfs::connect::IdentityStore::new(state.join("identity")) + rustfs::connect::IdentityStore::new(state.join("identity")) .load_or_create() .expect("enrolled identity"); @@ -347,7 +347,7 @@ fn production_cli_exports_top_net_and_fails_closed_for_unavailable_unsupported_a let net = top_command("net", &state, &net_output, "019e3ae0-0000-7000-8000-000000000021", 1, true) .output() .expect("run top.net"); - assert!(net.status.success(), "stderr: {}", String::from_utf8_lossy(&net.stderr)); + assert!(!net.status.success()); let stdout = String::from_utf8(net.stdout).expect("UTF-8 stdout"); let result: serde_json::Value = stdout .lines() @@ -355,38 +355,16 @@ fn production_cli_exports_top_net_and_fails_closed_for_unavailable_unsupported_a .map(|line| serde_json::from_str(line).expect("result JSON")) .expect("result line"); assert_eq!(result["toolId"], "top.net"); - assert_eq!(result["outcome"], "SUCCEEDED"); - assert_eq!(result["reasonCode"], "COMPLETE"); + assert_eq!(result["outcome"], "FAILED"); + assert_eq!(result["reasonCode"], "SOURCE_UNAVAILABLE"); assert_eq!(result["coverage"]["requestedUnits"], 1); - assert_eq!(result["coverage"]["completedUnits"], 1); + assert_eq!(result["coverage"]["completedUnits"], 0); assert_eq!(result["provenance"]["sourceCommit"], rustfs::version::build::COMMIT_HASH); assert_eq!( result["provenance"]["executableSha256"], sha256_file(Path::new(env!("CARGO_BIN_EXE_rustfs"))) ); - - let bytes = fs::read(&net_output).expect("top.net archive"); - #[cfg(unix)] - assert_eq!(fs::metadata(&net_output).expect("output metadata").permissions().mode() & 0o777, 0o600); - let mut archive = ZipArchive::new(Cursor::new(bytes)).expect("top.net archive"); - let names = (0..archive.len()) - .map(|index| archive.by_index(index).expect("archive member").name().to_owned()) - .collect::>(); - assert_eq!(names, ["envelope.json", "envelope.sig", "result.json"]); - - let envelope = archive_entry(&mut archive, "envelope.json"); - let signature_document = archive_entry(&mut archive, "envelope.sig"); - let signature_document: serde_json::Value = serde_json::from_slice(&signature_document).expect("signature JSON"); - let raw = URL_SAFE_NO_PAD - .decode_to_vec(signature_document["value"].as_str().expect("signature value")) - .expect("base64url signature"); - let signature = Signature::from_slice(&raw).expect("P-256 signature"); - let mut signed = b"rustfs-diagnostic-envelope-v1\0".to_vec(); - signed.extend_from_slice(&envelope); - VerifyingKey::from_public_key_der(&identity.public_key_der()) - .expect("public key") - .verify(&signed, &signature) - .expect("valid ES256 signature"); + assert!(!net_output.exists()); let locks_output = directory.path().join("locks.zip"); let locks = top_command("locks", &state, &locks_output, "019e3ae0-0000-7000-8000-000000000031", 1, true) diff --git a/rustfs/tests/connect_top_rpc.rs b/rustfs/tests/connect_top_rpc.rs index 251bac911..2f5cb9587 100644 --- a/rustfs/tests/connect_top_rpc.rs +++ b/rustfs/tests/connect_top_rpc.rs @@ -93,10 +93,14 @@ async fn top_rpc_captures_classified_rpc_outcomes_only() { assert_eq!(data.error_count, 1); assert_eq!(data.total_duration_micros, 18); let encoded = serde_json::to_value(data).expect("serialize top.rpc data"); - assert_eq!( - encoded.as_object().expect("top.rpc object").keys().collect::>(), - ["errorCount", "requestCount", "totalDurationMicros", "windowMillis"] - ); + let mut keys = encoded + .as_object() + .expect("top.rpc object") + .keys() + .cloned() + .collect::>(); + keys.sort(); + assert_eq!(keys, ["errorCount", "requestCount", "totalDurationMicros", "windowMillis"]); } #[tokio::test] diff --git a/rustfs/tests/connect_trace_record.rs b/rustfs/tests/connect_trace_record.rs index 0b0db58b3..dd00f8a55 100644 --- a/rustfs/tests/connect_trace_record.rs +++ b/rustfs/tests/connect_trace_record.rs @@ -14,11 +14,12 @@ use rustfs::connect::{ TelemetrySpanStatus, TelemetryTool, TraceRecordCompletion, TraceRecordLimits, encode_signed_telemetry_export, record_diagnostic_result, record_trace, record_trace_bus, save_signed_telemetry_export, }; +use rustfs::server::s3_http_request_guard; use rustfs_common::trace_bus::{ TelemetryTraceEvent, TelemetryTraceOperation, TelemetryTraceStatus, TraceEvent, TraceFunc, TraceKind, subscribe_trace_events, telemetry_trace_emit, telemetry_trace_subscriber_count, trace_emit, }; -use rustfs_io_metrics::{record_s3_op, s3_http_metrics::S3HttpRequestGuard}; +use rustfs_io_metrics::record_s3_op; use rustfs_s3_ops::S3Operation; use sha2::{Digest as _, Sha256}; use tokio::sync::mpsc; @@ -280,7 +281,7 @@ async fn connect_trace_record_uses_classified_s3_and_rpc_events_only() { .with_attr("error", "SYNTHETIC_SECRET_ERROR") })); - let mut s3_request = S3HttpRequestGuard::new("GET"); + let mut s3_request = s3_http_request_guard("GET"); s3_request.in_scope(|| record_s3_op(S3Operation::GetObject)); tokio::time::sleep(Duration::from_millis(1)).await; s3_request.response(200); diff --git a/scripts/check_s3s_footprint.sh b/scripts/check_s3s_footprint.sh index 1764417db..37eea95e0 100755 --- a/scripts/check_s3s_footprint.sh +++ b/scripts/check_s3s_footprint.sh @@ -61,8 +61,17 @@ cd "$(dirname "$0")/.." # 213 -> 212 on 2026-09-14: rustfs/backlog#1735 A4 moved rio's trailer # handle behind rustfs_rio::TrailerSource; the only adapter imports s3s through # the app storage_api shim, so crates/rio no longer references s3s. -S3S_IMPORT_FILES_BASELINE=209 -S3_ERROR_LINES_BASELINE=1588 +# 209 -> 210 and 1588 -> 1592 on 2026-09-14: #7785 landed the gateway key +# inventory admin handler (rustfs/src/admin/handlers/gateway_key_inventory.rs) +# for the RUSTFS_S3_STACK switch after the baselines were verified, leaving +# this guard red on main (measured 210/1592). The handler follows the house +# admin convention whose Operation::call signature is s3s-typed +# (S3Request/S3Result), so it cannot route through a non-s3s seam until the +# s3gate admin migration replaces the admin router (rustfs/backlog#1677 F1); +# no local refactor can shed the file-level import. Admit the measured +# growth: +1 direct-s3s admin file, +4 s3_error! invocation lines. +S3S_IMPORT_FILES_BASELINE=210 +S3_ERROR_LINES_BASELINE=1592 # ecstore-scoped ratchet (rustfs/backlog#1842): the storage engine must not # know S3 wire/DTO types (ARCHITECTURE.md invariant 4). The S3-*consuming* # client was extracted to crates/s3-client, where s3s usage is legitimate;