diff --git a/Cargo.lock b/Cargo.lock index 18b4dce5d..badc6a684 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,7 +299,7 @@ dependencies = [ "strum", "thiserror 2.0.21", "uuid", - "zstd 0.13.3", + "zstd 0.13.4", ] [[package]] @@ -492,7 +492,7 @@ dependencies = [ "arrow-select", "flatbuffers", "lz4_flex", - "zstd 0.13.3", + "zstd 0.13.4", ] [[package]] @@ -976,9 +976,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.128.0" +version = "1.129.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef519fda62fd0fdc066948f5ef33ecb8ae1bf0187bf1a12e8a775c93c4c47e7e" +checksum = "5c5cdb29ad25d71c9ee54c408a672c98d90a24a0ea6eb063025483e0e894b411" dependencies = [ "arc-swap", "aws-credential-types", @@ -1001,9 +1001,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.156.0" +version = "1.157.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1072c9aa20159b9b417115342d38c4c53108fc98e5359fcb86b7d4575747eb59" +checksum = "6ea9794c741520d8baec9462bcd0ad12164d4f5f78d008a8da9d6a0b7222caa8" dependencies = [ "arc-swap", "aws-credential-types", @@ -1037,9 +1037,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.118.0" +version = "1.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a89786db8ee6dff2c9302a354ef081354fcb0f527e16ee3cb249a6a5cb6df7" +checksum = "3c414f25ec4aa9dc18f5cc8f55e18848bb509fcda4ce7c77bee6072e95806bc9" dependencies = [ "arc-swap", "aws-credential-types", @@ -1062,9 +1062,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.120.0" +version = "1.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521c6a11a25f1244efddf947523f277a51a63f0a7ec228f0f62d826e88106366" +checksum = "bc638bb4355e6514334dcc3228cd4622f5ee54d704bd6a9010a5a91d7856d9d9" dependencies = [ "arc-swap", "aws-credential-types", @@ -1087,9 +1087,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.123.0" +version = "1.124.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8754d38d29448bd1d4aa4a6312ed010dfa296b3436eb2c551cc6e42fc6a66a38" +checksum = "3f8a0dd8fbde04ba58ad722113c7addb11bf94cf549b1ea0eb9ea427fb87198a" dependencies = [ "arc-swap", "aws-credential-types", @@ -3960,6 +3960,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", + "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -5295,9 +5296,9 @@ dependencies = [ [[package]] name = "hotpath" -version = "0.30.2" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39e941010d86063ff233bf8325269bf68097fb24ba67000fbddc0a410e8056e" +checksum = "d9632c1fd213e63c1b88d8f3e59543da4999b17f859cc1e1d67cf8f2207e8935" dependencies = [ "arc-swap", "async-channel", @@ -5330,15 +5331,15 @@ dependencies = [ [[package]] name = "hotpath-drain" -version = "0.30.2" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0faae1a9e1fd8ce5debaaaf452dcbe833fc8a8a4f3779d2ec8828b116c3c16a" +checksum = "738324f75aec847dea984f6806402f4f1bdfab437b4700fc8c3ebcad2dff6424" [[package]] name = "hotpath-macros" -version = "0.30.2" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24f5179ff680132003306354efd0ac8cde12e87b1a788719c836e91af35cb58" +checksum = "5d92fdfec803932e5764ea93c201cf7c6c2d8cff1871da44f64f5094c9c3ef8a" dependencies = [ "proc-macro2", "quote", @@ -5959,7 +5960,7 @@ dependencies = [ "lz4", "snap", "uuid", - "zstd 0.13.3", + "zstd 0.13.4", ] [[package]] @@ -6052,6 +6053,9 @@ name = "lazy_static" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin 0.9.9", +] [[package]] name = "lexical-core" @@ -6261,9 +6265,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" [[package]] name = "lru" @@ -6627,9 +6631,9 @@ dependencies = [ [[package]] name = "ml-kem" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ba01ebcf90f6aff8e9b9f55bb69b19e6250e4ade5787d0040eaa27dd226238" +checksum = "84ca78b66f46050d49fa21693ae8adab2081d8101b65d8067316b6b70b64a0d1" dependencies = [ "hybrid-array", "kem", @@ -6642,9 +6646,9 @@ dependencies = [ [[package]] name = "module-lattice" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe" +checksum = "c4a1e412c8ee83a1c678f427806219c49c96b2511b9f07497514f0e43fb26155" dependencies = [ "ctutils", "hybrid-array", @@ -6940,6 +6944,22 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -6964,6 +6984,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.2" @@ -7035,18 +7065,19 @@ dependencies = [ [[package]] name = "oauth2" -version = "5.1.0" -source = "git+https://github.com/ramosbugs/oauth2-rs?rev=89585f4a280afd7b285eaf1c9be0068874e03d22#89585f4a280afd7b285eaf1c9be0068874e03d22" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ "base64 0.22.1", "chrono", "getrandom 0.2.17", "http 1.5.0", - "rand 0.10.3", + "rand 0.8.8", "serde", "serde_json", "serde_path_to_error", - "sha2 0.11.1", + "sha2 0.10.9", "thiserror 1.0.69", "url", ] @@ -7208,29 +7239,31 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openidconnect" -version = "4.0.1" -source = "git+https://github.com/ramosbugs/openidconnect-rs?rev=2f77ee73bc4554e1cb6279adb386005429fb4e41#2f77ee73bc4554e1cb6279adb386005429fb4e41" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0d098a59353679624d406825940139e2b2d554a52b64c9fb1b1bc42a11fe5c" dependencies = [ "base64 0.22.1", "chrono", "dyn-clone", "ed25519-dalek 3.0.0", - "hmac 0.13.0", + "getrandom 0.4.3", + "hmac 0.12.1", "http 1.5.0", "itertools 0.14.0", "log", "oauth2", "p256", "p384", - "rand 0.10.3", - "rsa", + "rand 0.8.8", + "rsa 0.9.10", "serde", "serde-value", "serde_json", "serde_path_to_error", "serde_plain", "serde_with", - "sha2 0.11.1", + "sha2 0.10.9", "subtle", "thiserror 1.0.69", "url", @@ -7582,7 +7615,7 @@ dependencies = [ "snap", "tokio", "twox-hash", - "zstd 0.13.3", + "zstd 0.13.4", ] [[package]] @@ -7834,6 +7867,17 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + [[package]] name = "pkcs1" version = "0.8.0-rc.4" @@ -8932,9 +8976,9 @@ dependencies = [ [[package]] name = "reflink-copy" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9dd7ab4af0363d5ccfd2838d782a28196cf32a5cc2e4fe3c5dc83f2be588b8b" +checksum = "3dd31f458a66eeda05a9e837fc5f0effbaacbe8565868ecae5b37bf0544c19f5" dependencies = [ "cfg-if", "libc", @@ -9096,6 +9140,26 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1 0.7.5", + "pkcs8 0.10.2", + "rand_core 0.6.4", + "signature 2.2.0", + "spki 0.7.3", + "subtle", + "zeroize", +] + [[package]] name = "rsa" version = "0.10.0-rc.18" @@ -9106,7 +9170,7 @@ dependencies = [ "crypto-bigint", "crypto-primes", "digest 0.11.3", - "pkcs1", + "pkcs1 0.8.0-rc.4", "pkcs8 0.11.0", "rand_core 0.10.1", "sha2 0.11.1", @@ -9211,13 +9275,13 @@ dependencies = [ "p521", "pageant", "pbkdf2 0.13.0", - "pkcs1", + "pkcs1 0.8.0-rc.4", "pkcs5 0.8.1", "pkcs8 0.11.0", "polyval", "rand 0.10.3", "rand_core 0.10.1", - "rsa", + "rsa 0.10.0-rc.18", "russh-cryptovec", "russh-util", "salsa20 0.11.0", @@ -9409,7 +9473,7 @@ dependencies = [ "regex", "reqwest", "rmp-serde", - "rsa", + "rsa 0.10.0-rc.18", "rust-embed", "rustfs-audit", "rustfs-common", @@ -9612,7 +9676,7 @@ dependencies = [ "jsonwebtoken", "pbkdf2 0.13.0", "rand 0.10.3", - "rsa", + "rsa 0.10.0-rc.18", "serde", "serde_json", "sha2 0.11.1", @@ -12006,7 +12070,7 @@ dependencies = [ "p384", "p521", "rand_core 0.10.1", - "rsa", + "rsa 0.10.0-rc.18", "sec1", "serde", "sha1 0.11.0", @@ -13114,9 +13178,9 @@ dependencies = [ [[package]] name = "unicode-properties" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +checksum = "993ad287eb20e8ba755312801cfe1413e0f25d604075863acaddca10b04c1833" [[package]] name = "unicode-segmentation" @@ -14018,9 +14082,9 @@ dependencies = [ [[package]] name = "zip" -version = "9.1.3" +version = "9.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb" +checksum = "848dad21639597378c3be94c59fd4de683f0f6d8a5387575896e1245737d4f37" dependencies = [ "aes 0.9.3", "bzip2", @@ -14071,7 +14135,7 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.3" +version = "0.13.4" dependencies = [ "apache-avro", "arrow-array", diff --git a/Cargo.toml b/Cargo.toml index 3f712e555..2e67147c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -217,7 +217,7 @@ chacha20poly1305 = { version = "0.11.0" } crc-fast = "1.10.0" hmac = { version = "0.13.0" } jsonwebtoken = { version = "11.1.0" } -openidconnect = { default-features = false, version = "4.0" } +openidconnect = { default-features = false, version = "4.1.0" } pbkdf2 = "0.13.0" p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] } rsa = { version = "=0.10.0-rc.18" } @@ -253,9 +253,9 @@ atoi = "3.1.1" atomic_enum = "0.3.0" aws-config = { version = "1.13.0" } aws-credential-types = { version = "1.3.1" } -aws-sdk-kms = { default-features = false, version = "1.128.0" } -aws-sdk-s3 = { default-features = false, version = "1.156.0" } -aws-sdk-sts = { default-features = false, version = "1.123.0" } +aws-sdk-kms = { default-features = false, version = "1.129.0" } +aws-sdk-s3 = { default-features = false, version = "1.157.0" } +aws-sdk-sts = { default-features = false, version = "1.124.0" } aws-smithy-async = { version = "1.3.1" } aws-smithy-http-client = { default-features = false, version = "1.5.2" } aws-smithy-runtime-api = { version = "1.19.2" } @@ -356,7 +356,7 @@ winapi-util = "0.1.11" windows = { version = "0.62.2" } windows-sys = "0.61.2" xxhash-rust = { version = "0.8.19" } -zip = "9.1.3" +zip = "9.2.1" zstd = "0.14.3" # Observability and Metrics @@ -386,7 +386,7 @@ dav-server = "0.11.0" # Performance Analysis and Memory Profiling rustfs-mimalloc = { version = "0.7.2" } # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved. -hotpath = { version = "0.30.2", default-features = false } +hotpath = { version = "0.30.3", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.49" } @@ -427,7 +427,3 @@ strip = "none" # Pin upstream fixes until compatible crates.io releases are available. [patch.crates-io] zstd = { path = "crates/zstd-compat" } -# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain. -# Upstream: openidconnect-rs#244 and oauth2-rs#352. -openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" } -oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" } diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 6559c85b1..d0bcf863c 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -853,6 +853,8 @@ pub(crate) struct DiskHealthMetricEpoch { fallback_operation_metrics: StdRwLock>>, #[cfg(test)] multipart_cleanup_observer: std::sync::Mutex>>, + #[cfg(test)] + operation_timeout_override: std::sync::Mutex>, /// Caller API operations currently executing through the disk health wrapper. api_waiting: AtomicU32, /// Operations rejected because the disk was unavailable for the caller. @@ -880,6 +882,8 @@ impl Default for DiskHealthMetricEpoch { fallback_operation_metrics: StdRwLock::new(HashMap::new()), #[cfg(test)] multipart_cleanup_observer: std::sync::Mutex::new(None), + #[cfg(test)] + operation_timeout_override: std::sync::Mutex::new(None), api_waiting: AtomicU32::new(0), total_errors_availability: AtomicU64::new(0), total_errors_timeout: AtomicU64::new(0), @@ -1573,6 +1577,20 @@ impl Default for DiskHealthTracker { } } +#[cfg(test)] +#[derive(Debug)] +pub(crate) struct ScopedDiskOperationTimeout { + metrics: Arc, + previous: Option, +} + +#[cfg(test)] +impl Drop for ScopedDiskOperationTimeout { + fn drop(&mut self) { + *self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous; + } +} + /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities. /// This is similar to Go's xlStorageDiskIDCheck. #[derive(Debug, Clone)] @@ -1914,6 +1932,23 @@ impl LocalDiskWrapper { self.health_check } + #[cfg(test)] + pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout { + assert!(!timeout.is_zero(), "fault injection needs a positive timeout"); + // The metrics epoch follows this drive across clones and reconnects. + // Its scoped override cannot shorten unrelated healthy drives' I/O. + let previous = self + .metrics + .operation_timeout_override + .lock() + .expect("test timeout lock") + .replace(timeout); + ScopedDiskOperationTimeout { + metrics: Arc::clone(&self.metrics), + previous, + } + } + /// Enable health monitoring after disk creation. /// Used to defer health checks until after startup format loading completes. pub fn enable_health_check(&self) { @@ -2638,6 +2673,13 @@ impl LocalDiskWrapper { } return result; } + #[cfg(test)] + let timeout_duration = self + .metrics + .operation_timeout_override + .lock() + .expect("test timeout lock") + .map_or(timeout_duration, |injected| timeout_duration.min(injected)); // Execute the operation with timeout let result = tokio::time::timeout(timeout_duration, operation).await; @@ -6044,6 +6086,67 @@ mod tests { assert!(health.offline_duration().is_none()); } + #[tokio::test] + async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() { + let directories = [ + tempfile::tempdir().expect("faulted drive"), + tempfile::tempdir().expect("healthy drive"), + ]; + let mut wrappers = Vec::new(); + for directory in &directories { + let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive")); + wrappers.push(LocalDiskWrapper::new(disk, false)); + } + let reconnected = LocalDiskWrapper::new_with_reconnect_state( + Arc::clone(&wrappers[0].disk), + false, + Some(wrappers[0].health_tracker_epoch_for_reconnect()), + ); + let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1)); + tokio::time::pause(); + let operation = || async { + tokio::time::sleep(Duration::from_secs(2)).await; + Ok(()) + }; + let (faulted, healthy) = tokio::join!( + reconnected.track_disk_health_with_op_and_timeout_action( + "read_metadata", + operation, + Duration::from_secs(3), + TimeoutHealthAction::IgnoreFailure, + ), + wrappers[1].track_disk_health_with_op_and_timeout_action( + "read_metadata", + operation, + Duration::from_secs(3), + TimeoutHealthAction::IgnoreFailure, + ), + ); + assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout); + healthy.expect("healthy drive keeps its caller's deadline"); + let shorter_started = tokio::time::Instant::now(); + let shorter = wrappers[0] + .track_disk_health_with_op_and_timeout_action( + "read_metadata", + operation, + Duration::from_millis(500), + TimeoutHealthAction::IgnoreFailure, + ) + .await; + assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout); + assert!(shorter_started.elapsed() < Duration::from_secs(1)); + wrappers[0] + .track_disk_health_with_op("read_metadata", operation, Duration::ZERO) + .await + .expect("explicit unbounded physical ownership cannot acquire a test deadline"); + drop(scope); + reconnected + .track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3)) + .await + .expect("dropping the scope restores reconnected callers' deadlines"); + } + #[tokio::test] async fn ignored_timeout_does_not_mark_drive_failure() { let dir = tempfile::tempdir().expect("temp dir should be created"); diff --git a/crates/ecstore/src/services/rebalance/worker.rs b/crates/ecstore/src/services/rebalance/worker.rs index 34ed357c2..7111dc538 100644 --- a/crates/ecstore/src/services/rebalance/worker.rs +++ b/crates/ecstore/src/services/rebalance/worker.rs @@ -318,6 +318,21 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool { } } +fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool { + match rebalance_error_source(err) { + Error::Lock(err) => is_rebalance_transient_lock_error(err), + // No metadata was written when lock admission failed. Each attempt + // must acquire its quorum again before loading or saving this run. + Error::NamespaceLockQuorumUnavailable { + mode: "write", + bucket, + object, + .. + } => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME, + _ => false, + } +} + fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool { if err.kind() == std::io::ErrorKind::TimedOut { return true; @@ -385,9 +400,7 @@ where match result { Ok(value) => return Ok(value), Err(err) => { - if attempt.saturating_add(1) >= max_attempts.max(1) - || !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err)) - { + if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) { return Err(err); } #[cfg(test)] @@ -719,6 +732,82 @@ mod error_source_tests { } } + #[tokio::test(start_paused = true)] + async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() { + let mut attempts = 0; + retry_rebalance_metadata_access(None, 3, || { + attempts += 1; + std::future::ready(if attempts == 1 { + resolve_rebalance_meta_save_result( + Err(rebalance_meta_lock_error( + rustfs_lock::LockError::QuorumNotReached { + required: 1, + achieved: 0, + }, + "write", + )), + "stop_rebalance", + ) + } else { + Ok(()) + }) + }) + .await + .expect("a later quorum-protected admission must be allowed to succeed"); + assert_eq!(attempts, 2); + } + + #[tokio::test(start_paused = true)] + async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() { + for max_attempts in [0, 1, 3] { + let mut attempts = 0; + let result = retry_rebalance_metadata_access(None, max_attempts, || { + attempts += 1; + std::future::ready(Err::<(), _>(rebalance_meta_lock_error( + rustfs_lock::LockError::QuorumNotReached { + required: 1, + achieved: 0, + }, + "write", + ))) + }) + .await; + let error = result.expect_err("exhausted admission attempts must never acknowledge a write"); + assert_eq!(attempts, max_attempts.max(1)); + assert!(matches!( + rebalance_error_source(&error), + Error::NamespaceLockQuorumUnavailable { + mode: "write", + required: 1, + achieved: 0, + .. + } + )); + } + } + + #[tokio::test(start_paused = true)] + async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() { + let mut attempts = 0; + let result = retry_rebalance_metadata_access(None, 3, || { + attempts += 1; + std::future::ready(Err::<(), _>(if attempts == 1 { + rebalance_meta_lock_error( + rustfs_lock::LockError::QuorumNotReached { + required: 1, + achieved: 0, + }, + "write", + ) + } else { + Error::OperationCanceled + })) + }) + .await; + assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt"); + assert!(matches!(result, Err(Error::OperationCanceled))); + } + #[tokio::test] async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() { for err in [ @@ -731,6 +820,20 @@ mod error_source_tests { required: 3, achieved: 2, }, + Error::NamespaceLockQuorumUnavailable { + mode: "write", + bucket: "data-bucket".to_string(), + object: REBAL_META_NAME.to_string(), + required: 1, + achieved: 0, + }, + Error::NamespaceLockQuorumUnavailable { + mode: "write", + bucket: crate::disk::RUSTFS_META_BUCKET.to_string(), + object: "other-metadata.bin".to_string(), + required: 1, + achieved: 0, + }, Error::other("stale rebalance run rejected: lock acquisition timed out"), Error::other("rebalance distributed run fence lost"), Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)), diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 71bcc110f..e6693783d 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -10060,7 +10060,7 @@ mod tests { // Real disk callbacks and health probes must share wall-clock deadlines. // Advancing Tokio time can expire unrelated healthy-drive probes. - temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async { + temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async { let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; let bucket = "mpu-quorum-success-physical"; @@ -10087,6 +10087,8 @@ mod tests { local.observe_multipart_cleanup_for_test(&upload_path) }).collect::>(); let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") }; + // Fault only the held leg; healthy disks keep their normal I/O budgets. + let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2)); let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target"); let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key"); let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); @@ -10243,7 +10245,15 @@ mod tests { let size = copy_info.get_actual_size().expect("actual source size"); copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream"))); tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default())) - .await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject"); + .await.expect("cross-key Copy cannot wait on the old successful logical owner") + .unwrap_or_else(|error| { + let health = disks.iter().enumerate().map(|(slot, disk)| { + let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") }; + let metrics = local.metrics_snapshot(); + (slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls) + }).collect::>(); + panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}"); + }); drop(copy_info); let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body"); let mut actual = Vec::new(); diff --git a/crates/iam/src/oidc/provider.rs b/crates/iam/src/oidc/provider.rs index 6b4f083f8..4ab398762 100644 --- a/crates/iam/src/oidc/provider.rs +++ b/crates/iam/src/oidc/provider.rs @@ -16,9 +16,10 @@ use super::{ transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics}, }; use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC}; -use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm}; +use openidconnect::core::{CoreIdToken, CoreIdTokenClaims, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm}; use openidconnect::{ - AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl, + AsyncHttpClient, ClaimsVerificationError, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, NonceVerifier, + ProviderMetadataWithLogout, SignatureVerificationError, TokenUrl, }; use serde::Deserialize; use std::collections::HashMap; @@ -212,6 +213,35 @@ impl DiscoveredProviderMetadata { } } +pub(super) fn verify_id_token_claims<'a, N: NonceVerifier>( + token: &'a CoreIdToken, + verifier: &CoreIdTokenVerifier<'_>, + nonce_verifier: N, +) -> Result<&'a CoreIdTokenClaims, ClaimsVerificationError> { + let expected_length = match token.signing_alg().map_err(ClaimsVerificationError::SignatureVerification)? { + CoreJwsSigningAlgorithm::HmacSha256 => 32, + CoreJwsSigningAlgorithm::HmacSha384 => 48, + CoreJwsSigningAlgorithm::HmacSha512 => 64, + _ => return token.claims(verifier, nonce_verifier), + }; + // RUSTFS_COMPAT_TODO(openidconnect-hmac-signature-width): Reject widths that upstream converts to fixed-size arrays. Remove after upstream returns verification errors for arbitrary HMAC signature lengths. + // openidconnect 4.1.0 converts HMAC signatures to fixed-size arrays before verifying them. + let invalid_length = || { + ClaimsVerificationError::SignatureVerification(SignatureVerificationError::CryptoError( + "invalid HMAC signature length".to_owned(), + )) + }; + let compact = token.to_string(); + let (_, signature) = compact.rsplit_once('.').ok_or_else(invalid_length)?; + let signature = base64_simd::URL_SAFE_NO_PAD + .decode_to_vec(signature) + .map_err(|_| invalid_length())?; + if signature.len() != expected_length { + return Err(invalid_length()); + } + token.claims(verifier, nonce_verifier) +} + // This adapter is used only for discovery/JWKS fetches, never token exchange. struct JwksAcceptClient<'a> { inner: &'a ReqwestHttpClient, diff --git a/crates/iam/src/oidc/provider_tests.rs b/crates/iam/src/oidc/provider_tests.rs index 8aa240412..ba46832dd 100644 --- a/crates/iam/src/oidc/provider_tests.rs +++ b/crates/iam/src/oidc/provider_tests.rs @@ -242,7 +242,7 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() { let now = time::OffsetDateTime::now_utc().unix_timestamp(); let payload = serde_json::json!({ "iss": config.config_url, "sub": "synthetic-subject", "aud": config.client_id, - "iat": now, "exp": now + 300, + "iat": now, "exp": now + 300, "nonce": "synthetic-nonce", }); for (algorithm, name, signature_length) in [ @@ -275,22 +275,28 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() { .expect("parse configured provider metadata"); let verifier = metadata.verifier(&config); assert!( - valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(), + verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(), "valid {name} token must pass for workload={workload}" ); + assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("synthetic-nonce".into())).is_ok()); + assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("wrong-nonce".into())).is_err()); for length in [1, 0, signature_length - 1, signature_length, signature_length + 1, 128] { let signature = base64_simd::URL_SAFE_NO_PAD.encode_to_string(vec![0_u8; length]); let malformed: CoreIdToken = format!("{signing_input}.{signature}") .parse() - .expect("invalid signature length must still reach cryptographic verification"); + .expect("invalid signature length must remain parseable before verification"); assert!( - malformed.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(), + verify_id_token_claims(&malformed, &verifier, |_: Option<&Nonce>| Ok(())).is_err(), "{name} signature length {length} must return an error for workload={workload}" ); + assert!( + verify_id_token_claims(&malformed, &verifier, &Nonce::new("synthetic-nonce".into())).is_err(), + "{name} signature length {length} must return an error with a nonce for workload={workload}" + ); } assert!( - valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(), + verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(), "invalid inputs must not prevent later valid {name} verification" ); } diff --git a/crates/iam/src/oidc/runtime.rs b/crates/iam/src/oidc/runtime.rs index 7088d31d9..3a41e84d7 100644 --- a/crates/iam/src/oidc/runtime.rs +++ b/crates/iam/src/oidc/runtime.rs @@ -16,7 +16,7 @@ use super::{ OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot, SourcedOidcProviderConfig, load_oidc_config_snapshot, }, - provider::{ProviderRuntime, discover_provider_for_recovery}, + provider::{ProviderRuntime, discover_provider_for_recovery, verify_id_token_claims}, state::{OidcAuthSession, OidcLogoutSession, OidcStateStore}, transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics}, }; @@ -535,7 +535,7 @@ impl OidcSys { let verifier = client .id_token_verifier() .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone())); + let verified = verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone())); if let Err(e) = verified { let verification_error = e.to_string(); warn!( @@ -583,8 +583,7 @@ impl OidcSys { let verifier = client .id_token_verifier() .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - id_token - .claims(&verifier, &Nonce::new(session.nonce.clone())) + verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone())) .map_err(|retry_err| { error!( event = EVENT_OIDC_DIAGNOSTICS, @@ -767,7 +766,7 @@ impl OidcSys { .metadata .verifier(&config) .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) { + if let Err(e) = verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) { state = self .provider_runtime .refresh_provider_state(&provider_id, &config) @@ -780,8 +779,7 @@ impl OidcSys { .metadata .verifier(&config) .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - id_token - .claims(&verifier, |_: Option<&Nonce>| Ok(())) + verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) .map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?; } diff --git a/crates/zstd-compat/Cargo.toml b/crates/zstd-compat/Cargo.toml index 53f711fcb..b0f63dbf9 100644 --- a/crates/zstd-compat/Cargo.toml +++ b/crates/zstd-compat/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zstd" -version = "0.13.3" +version = "0.13.4" publish = false edition.workspace = true license.workspace = true diff --git a/crates/zstd-compat/README.md b/crates/zstd-compat/README.md index d72d534f1..d54a3defa 100644 --- a/crates/zstd-compat/README.md +++ b/crates/zstd-compat/README.md @@ -6,6 +6,6 @@ This private package satisfies the version constraints of Avro, Arrow IPC, Parqu The fixed implementation closes RUSTSEC-2026-0391, RUSTSEC-2026-0398, and RUSTSEC-2026-0403. A successful advisory scan alone is insufficient evidence: cargo-deny skips local path packages. The dependency-graph test therefore requires this adapter's sole runtime dependency to be the exact fixed registry implementation and rejects vulnerable registry codec versions. Security tests exercise abandoned stream contexts, malformed legacy frames, and seekable offsets past EOF. Consumer tests retain zstandard support in all four upstream formats. -Run `cargo nextest run --locked -p zstd@0.13.3` for the focused tests. The seekable feature is enabled by a test-only dependency to cover the optional vulnerable API without enabling it in production. +Run `cargo nextest run --locked -p zstd@0.13.4` for the focused tests. The seekable feature is enabled by a test-only dependency to cover the optional vulnerable API without enabling it in production. Remove this package and the root Cargo patch when all consumers accept the fixed zstd release line directly. The cleanup marker is registered in the architecture compatibility register. diff --git a/crates/zstd-compat/tests/dependency_graph.rs b/crates/zstd-compat/tests/dependency_graph.rs index d9db2dd5c..44df19810 100644 --- a/crates/zstd-compat/tests/dependency_graph.rs +++ b/crates/zstd-compat/tests/dependency_graph.rs @@ -57,7 +57,7 @@ fn legacy_constraints_resolve_only_to_the_fixed_registry_implementation() { let version = Version::parse(package["version"].as_str().expect("package version")).expect("valid package semver"); if name == "zstd" && package.get("source").is_none() { adapters += 1; - assert_eq!(version, Version::new(0, 13, 3)); + assert_eq!(version, Version::new(0, 13, 4)); let runtime = package["dependencies"].as_array().expect("adapter dependency edges"); assert!(runtime.iter().any(|value| value.as_str() == Some("zstd 0.14.3"))); } else { diff --git a/deny.toml b/deny.toml index 0969f7e49..f850a47f1 100644 --- a/deny.toml +++ b/deny.toml @@ -36,10 +36,6 @@ allow-git = [ # Official s3s repository pinned for Iceberg SigV4 path encoding support. # owner: GatewayJ review: 2026-10 "https://github.com/s3s-project/s3s", - # Upstream OIDC/OAuth2 crypto migrations, pinned to reviewed PR commits. - # owner: rustfs-maintainers review: 2026-10 - "https://github.com/ramosbugs/openidconnect-rs", - "https://github.com/ramosbugs/oauth2-rs", # Completion ownership patch fixed to rustfs/uring#20. # owner: overtrue review: 2026-10 "https://github.com/rustfs/uring.git", diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 8ac4bf7c5..6dafa8410 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -11,6 +11,8 @@ ## Open Items +- `openidconnect-hmac-signature-width` ID token verification guard: openidconnect 4.1.0 converts decoded HMAC signatures to fixed-size arrays, which panic on unexpected lengths. Check HS256, HS384, and HS512 signature widths before delegating all cryptographic, issuer, audience, expiry, and nonce checks to the upstream verifier. Remove the guard after a supported upstream release returns verification errors for arbitrary HMAC signature lengths. + - `zstd-013-fixed-backend` dependency constraint bridge: Avro, Arrow IPC, Parquet, and Kafka still require zstd 0.13. The local adapter re-exports the fixed registry zstd 0.14.3 implementation and its zstd-safe types. Remove the adapter and Cargo patch after every dependency accepts the fixed zstd line without the 0.13 bridge. - `8565` historical rebalance replacement recovery: older schema-seven intents can exhaust their retry budget before format repair when retained rebalance metadata describes removed pools. After validating blank targets, terminal historical topology, and ownership, grant one durable handoff and one rebuild attempt. Remove after supported direct upgrades no longer admit snapshots lacking replacement_preformat_handoff_retry_done and all retained affected intents have been resolved. diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 89b09cd59..809ea5e1c 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -273,9 +273,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.156.0" +version = "1.157.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1072c9aa20159b9b417115342d38c4c53108fc98e5359fcb86b7d4575747eb59" +checksum = "6ea9794c741520d8baec9462bcd0ad12164d4f5f78d008a8da9d6a0b7222caa8" dependencies = [ "arc-swap", "aws-credential-types", @@ -1837,18 +1837,18 @@ dependencies = [ [[package]] name = "hotpath" -version = "0.30.2" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39e941010d86063ff233bf8325269bf68097fb24ba67000fbddc0a410e8056e" +checksum = "d9632c1fd213e63c1b88d8f3e59543da4999b17f859cc1e1d67cf8f2207e8935" dependencies = [ "hotpath-macros", ] [[package]] name = "hotpath-macros" -version = "0.30.2" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24f5179ff680132003306354efd0ac8cde12e87b1a788719c836e91af35cb58" +checksum = "5d92fdfec803932e5764ea93c201cf7c6c2d8cff1871da44f64f5094c9c3ef8a" [[package]] name = "http" @@ -2418,9 +2418,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" [[package]] name = "lru" @@ -4134,8 +4134,7 @@ dependencies = [ [[package]] name = "rustfs-uring" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7deda770b7981ee3bfaa202ecd18625fc794c962b8d58cd4f436952b7e66dcf" +source = "git+https://github.com/rustfs/uring.git?rev=eb8fc160e71d804427de1e905f5c226d580f71bf#eb8fc160e71d804427de1e905f5c226d580f71bf" dependencies = [ "io-uring", "libc", diff --git a/rustfs/tests/connect_perf_object.rs b/rustfs/tests/connect_perf_object.rs index 895df2d63..bcd56fb15 100644 --- a/rustfs/tests/connect_perf_object.rs +++ b/rustfs/tests/connect_perf_object.rs @@ -540,7 +540,7 @@ struct CleanupObservation { elapsed: Duration, } -async fn observe_cleanup(replies: [CleanupReply; 2]) -> CleanupObservation { +async fn observe_cleanup(replies: [CleanupReply; 2], measurement_delay: Duration) -> CleanupObservation { use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; const PRIVATE_BODY: &str = "private-cleanup-response-body"; @@ -583,6 +583,12 @@ async fn observe_cleanup(replies: [CleanupReply; 2]) -> CleanupObservation { .to_owned(); let reply = if index < 2 { assert!(request_line.starts_with("PUT ")); + if index == 1 && !measurement_delay.is_zero() { + // Model descheduling during the measured PUT without a wall-clock sleep. + tokio::time::pause(); + tokio::time::advance(measurement_delay).await; + tokio::time::resume(); + } CleanupReply::Status(200) } else { assert!(request_line.starts_with("DELETE ")); @@ -616,6 +622,8 @@ async fn observe_cleanup(replies: [CleanupReply; 2]) -> CleanupObservation { ) .expect("cleanup probe"); let mut request = request(ObjectOperation::PutObject); + // Status fixtures must reach cleanup independently of short measurement deadlines. + request.duration = REAL_SERVER_MEASUREMENT_BUDGET; request.traffic_bytes = 4; let log = CleanupLog::default(); let writer = log.clone(); @@ -707,7 +715,7 @@ async fn cleanup_diagnostics_identify_each_failed_http_stage_without_private_con ([CleanupReply::Status(204), CleanupReply::Status(403)], "bucket_delete", 403), ([CleanupReply::Status(204), CleanupReply::Status(404)], "bucket_delete", 404), ] { - let observed = observe_cleanup(replies).await; + let observed = observe_cleanup(replies, Duration::ZERO).await; assert_cleanup_failure(&observed, stage, "CleanupFailed", Some(status)); assert_eq!(observed.deletes.len(), if stage == "object_delete" { 1 } else { 2 }); } @@ -720,7 +728,7 @@ async fn cleanup_diagnostics_identify_each_disconnected_transport_stage() { ([CleanupReply::Disconnect, CleanupReply::Status(204)], "object_delete"), ([CleanupReply::Status(204), CleanupReply::Disconnect], "bucket_delete"), ] { - let observed = observe_cleanup(replies).await; + let observed = observe_cleanup(replies, Duration::ZERO).await; assert_cleanup_failure(&observed, stage, "ProtocolFailure", None); } } @@ -728,13 +736,21 @@ async fn cleanup_diagnostics_identify_each_disconnected_transport_stage() { #[tokio::test] async fn cleanup_accepts_missing_object_and_still_deletes_the_bucket() { let _guard = TEST_LOCK.lock().await; - let observed = observe_cleanup([CleanupReply::Status(404), CleanupReply::Status(204)]).await; - assert_eq!(observed.measurement.result.outcome(), ObjectOutcome::Succeeded); - assert_eq!(observed.measurement.target.reason_code, ObjectTargetReasonCode::Complete); - assert_eq!(observed.deletes.len(), 2); - assert!(observed.deletes[0].ends_with("/synthetic-object HTTP/1.1")); - assert!(observed.deletes[1].ends_with("/ HTTP/1.1")); - assert!(!observed.log.contains("connect_object_cleanup_failed")); + for measurement_delay in [Duration::ZERO, Duration::from_secs(2)] { + let observed = observe_cleanup([CleanupReply::Status(404), CleanupReply::Status(204)], measurement_delay).await; + assert_eq!( + observed.measurement.result.outcome(), + ObjectOutcome::Succeeded, + "target result: {:?}; cleanup diagnostics: {}", + observed.measurement.target, + observed.log + ); + assert_eq!(observed.measurement.target.reason_code, ObjectTargetReasonCode::Complete); + assert_eq!(observed.deletes.len(), 2); + assert!(observed.deletes[0].ends_with("/synthetic-object HTTP/1.1")); + assert!(observed.deletes[1].ends_with("/ HTTP/1.1")); + assert!(!observed.log.contains("connect_object_cleanup_failed")); + } } #[tokio::test] @@ -750,7 +766,7 @@ async fn cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_dead "bucket_delete", ), ] { - let observed = observe_cleanup(replies).await; + let observed = observe_cleanup(replies, Duration::ZERO).await; assert_cleanup_failure(&observed, stage, "TimedOut", None); assert!(observed.elapsed >= Duration::from_secs(5)); assert!(observed.log.contains("remaining_ms=0"), "{}", observed.log);