gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 05:06:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix/b5-t4-b2-remote-target-serde

Browse Source
This commit is contained in:
houseme
2026-08-23 12:28:53 +08:00
committed by GitHub
parent cddc003c16 b6ba89d9e4
commit 6264314cdc
254 changed files with 22099 additions and 4287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/AGENTS.md
+6 -2
View File
@@ -19,7 +19,9 @@ Applies to all paths under `crates/`.
- Document lock acquisition order when a module uses multiple locks. Never acquire the same set of locks in different orders across code paths.
- Never hold a `tokio::sync::RwLock`/`Mutex` write guard across `.await` points unless the critical section is unavoidably async and the hold time is bounded.
- Prefer `compare_exchange` loops over load-then-store for concurrent counters (peak values, adaptive heuristics).
- Prefer direct atomic `fetch_*` operations for unconditional updates and
`compare_exchange` loops only for conditional updates such as peaks or
adaptive state.
- When resetting multi-field atomic statistics, use a version/sequence counter or accept that concurrent readers may see partial snapshots; document the tradeoff.
- `std::sync::Mutex` is acceptable in async context only when held for a brief, non-`await`-containing critical section. If in doubt, use `tokio::sync::Mutex`.
@@ -40,7 +42,9 @@ Applies to all paths under `crates/`.
- Keep unit tests close to the module they test.
- Keep integration tests under each crate's `tests/` directory.
- Add regression tests for bug fixes and behavior changes.
- Every test function must contain at least one `assert!`/`assert_eq!`/`assert_matches!`. A test that only calls code without asserting is not a test.
- Every test needs an observable failure criterion. Direct assertions,
delegated assertions, snapshots/properties, `#[should_panic]`, and meaningful
`Result` failures are all valid; a call that can silently succeed is not.
- In tests, prefer `.expect("context: what was being tested")` over bare `.unwrap()`. A test failure should tell you which operation failed and with what input.
## Async and Performance
crates/audit/AGENTS.md
-1
View File
@@ -50,4 +50,3 @@ crate.
- `cargo test -p rustfs-audit`
- Focused: `cargo test -p rustfs-audit --test pipeline_layer_test`
- Focused: `cargo test -p rustfs-audit pipeline`
- Full gate before commit: `make pre-commit`
crates/common/src/metrics.rs
+56
View File
@@ -901,6 +901,10 @@ pub struct Metrics {
scanner_cycle_max_duration_millis: AtomicU64,
scanner_cycle_max_objects: AtomicU64,
scanner_cycle_max_directories: AtomicU64,
scanner_cycle_timeout_total: AtomicU64,
scanner_cycle_recovery_required_total: AtomicU64,
scanner_cycle_last_progress_age_seconds: AtomicU64,
scanner_leader_lease_without_progress: AtomicBool,
scanner_bitrot_cycle_enabled: AtomicBool,
scanner_bitrot_cycle_millis: AtomicU64,
scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
@@ -1370,6 +1374,14 @@ pub struct ScannerMetricsReport {
#[serde(default)]
pub cycle_max_directories: u64,
#[serde(default)]
pub cycle_timeout_total: u64,
#[serde(default)]
pub cycle_recovery_required_total: u64,
#[serde(default)]
pub cycle_last_progress_age: u64,
#[serde(default)]
pub leader_lease_without_progress: bool,
#[serde(default)]
pub bitrot_cycle_enabled: bool,
#[serde(default)]
pub bitrot_cycle_seconds: f64,
@@ -1430,6 +1442,9 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total
const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
fn scan_cycle_result_label(result: u8) -> &'static str {
match result {
@@ -1913,6 +1928,10 @@ impl Metrics {
scanner_cycle_max_duration_millis: AtomicU64::new(0),
scanner_cycle_max_objects: AtomicU64::new(0),
scanner_cycle_max_directories: AtomicU64::new(0),
scanner_cycle_timeout_total: AtomicU64::new(0),
scanner_cycle_recovery_required_total: AtomicU64::new(0),
scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
scanner_leader_lease_without_progress: AtomicBool::new(false),
scanner_bitrot_cycle_enabled: AtomicBool::new(false),
scanner_bitrot_cycle_millis: AtomicU64::new(0),
scanner_checkpoint: Mutex::new(None),
@@ -2412,12 +2431,29 @@ impl Metrics {
.store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
self.scanner_cycle_max_directories
.store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
self.scanner_bitrot_cycle_enabled
.store(bitrot_cycle.is_some(), Ordering::Relaxed);
self.scanner_bitrot_cycle_millis
.store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
}
pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
if recovery_required {
self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
}
self.scanner_cycle_last_progress_age_seconds
.store(progress_age.as_secs(), Ordering::Relaxed);
self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
}
pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
if let Some(concurrency_limit) = concurrency_limit {
self.scanner_set_scan_concurrency_limit
@@ -3265,6 +3301,10 @@ impl Metrics {
m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.scan_checkpoint = match self.scanner_checkpoint.lock() {
@@ -4926,4 +4966,20 @@ mod tests {
assert!(!report.bitrot_cycle_enabled);
assert_eq!(report.bitrot_cycle_seconds, 0.0);
}
#[tokio::test]
async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
let metrics = Metrics::new();
metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
let timed_out = metrics.report().await;
assert_eq!(timed_out.cycle_timeout_total, 1);
assert_eq!(timed_out.cycle_last_progress_age, 17);
assert!(timed_out.leader_lease_without_progress);
metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
let current = metrics.report().await;
assert_eq!(current.cycle_timeout_total, 1);
assert_eq!(current.cycle_last_progress_age, 0);
assert!(!current.leader_lease_without_progress);
}
}
crates/config/README.md
+6
View File
@@ -84,6 +84,12 @@ Current guidance:
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
Scanner cycle budget controls:
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
## Mmap read environment aliases
- `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical)
crates/config/src/constants/runtime.rs
+7
View File
@@ -57,6 +57,13 @@ pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024;
pub const DEFAULT_EVENT_INTERVAL: u32 = 61;
pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
/// Dedicated blocking thread pool for fsync/fdatasync operations.
/// When > 1, fsync operations are isolated from the main blocking pool to
/// prevent device-bound fsync from starving read operations (pread/stat/open).
/// Default 0 means auto (no isolation, use main runtime).
pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
// Dial9 Tokio Telemetry Default values
pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry";
crates/config/src/constants/scanner.rs
+6 -3
View File
@@ -143,9 +143,12 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
/// Default scanner speed preset.
pub const DEFAULT_SCANNER_SPEED: &str = "default";
/// Default scanner cycle runtime budget.
/// `0` keeps the existing unbounded per-cycle behavior.
pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0;
/// Default scanner cycle runtime budget when no override is configured.
///
/// An explicit `0` remains the compatibility escape hatch for an unbounded
/// cycle. Keeping the unset default finite prevents a stalled scanner I/O
/// operation from holding the leader lease forever.
pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60;
/// Default scanner per-cycle object budget.
/// `0` keeps the existing unbounded per-cycle behavior.
crates/data-usage/src/data_usage.rs
+64 -18
View File
@@ -585,9 +585,12 @@ impl VersionsHistogram {
}
}
/// Replication statistics for a single target
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct ReplicationStats {
/// Replication statistics for a single target.
///
/// Renamed from `ReplicationStats`; serde field names are preserved
/// byte-identically to maintain wire compatibility with existing snapshots.
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ReplicationTargetUsage {
pub pending_size: u64,
pub replicated_size: u64,
pub failed_size: u64,
@@ -600,7 +603,7 @@ pub struct ReplicationStats {
pub replicated_count: u64,
}
impl ReplicationStats {
impl ReplicationTargetUsage {
pub fn is_empty(&self) -> bool {
let Self {
pending_size,
@@ -636,7 +639,7 @@ impl ReplicationStats {
/// Replication statistics for all targets
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct ReplicationAllStats {
pub targets: HashMap<String, ReplicationStats>,
pub targets: HashMap<String, ReplicationTargetUsage>,
pub replica_size: u64,
pub replica_count: u64,
}
@@ -649,7 +652,7 @@ impl ReplicationAllStats {
targets,
} = self;
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationTargetUsage::is_empty)
}
#[deprecated(note = "use is_empty instead")]
@@ -2466,7 +2469,7 @@ mod tests {
#[test]
fn replication_stats_empty_checks_every_field() {
type SetField = fn(&mut ReplicationStats);
type SetField = fn(&mut ReplicationTargetUsage);
let cases: [(&str, SetField); 10] = [
("pending_size", |stats| stats.pending_size = 1),
@@ -2481,9 +2484,9 @@ mod tests {
("replicated_count", |stats| stats.replicated_count = 1),
];
assert!(ReplicationStats::default().is_empty());
assert!(ReplicationTargetUsage::default().is_empty());
for (field, set_nonzero) in cases {
let mut stats = ReplicationStats::default();
let mut stats = ReplicationTargetUsage::default();
set_nonzero(&mut stats);
assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
}
@@ -2514,17 +2517,17 @@ mod tests {
}
let empty_targets = ReplicationAllStats {
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationTargetUsage::default())]),
..Default::default()
};
assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
let stats = ReplicationAllStats {
targets: HashMap::from([
("arn:test:empty".to_string(), ReplicationStats::default()),
("arn:test:empty".to_string(), ReplicationTargetUsage::default()),
(
"arn:test:non-empty".to_string(),
ReplicationStats {
ReplicationTargetUsage {
pending_count: 1,
..Default::default()
},
@@ -2565,7 +2568,7 @@ mod tests {
replication_stats: Some(ReplicationAllStats {
targets: HashMap::from([(
"arn:test:pending".to_string(),
ReplicationStats {
ReplicationTargetUsage {
pending_count: 1,
..Default::default()
},
@@ -2714,7 +2717,7 @@ mod tests {
targets: HashMap::from([
(
"arn:self-only".to_string(),
ReplicationStats {
ReplicationTargetUsage {
pending_size: 7,
pending_count: 1,
..Default::default()
@@ -2722,7 +2725,7 @@ mod tests {
),
(
"arn:shared".to_string(),
ReplicationStats {
ReplicationTargetUsage {
failed_size: 3,
failed_count: 1,
missed_threshold_size: 2,
@@ -2741,7 +2744,7 @@ mod tests {
targets: HashMap::from([
(
"arn:shared".to_string(),
ReplicationStats {
ReplicationTargetUsage {
failed_size: 5,
failed_count: 2,
after_threshold_size: 4,
@@ -2751,7 +2754,7 @@ mod tests {
),
(
"arn:other-only".to_string(),
ReplicationStats {
ReplicationTargetUsage {
replicated_size: 11,
replicated_count: 3,
..Default::default()
@@ -2993,7 +2996,9 @@ mod tests {
fn replication_target_deserialization_preserves_large_historical_maps() {
let mut stats = ReplicationAllStats::default();
for index in 0..=1024 {
stats.targets.insert(format!("target-{index}"), ReplicationStats::default());
stats
.targets
.insert(format!("target-{index}"), ReplicationTargetUsage::default());
}
let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode");
let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded)
@@ -3002,6 +3007,47 @@ mod tests {
assert_eq!(decoded.targets.len(), stats.targets.len());
}
/// Round-trip test: encoding a [`ReplicationTargetUsage`] and decoding it back
/// must produce the exact same value. This guards against accidental serde
/// field-name drift during the `ReplicationStats` -> `ReplicationTargetUsage`
/// rename. Wire-level field names are the serialized Rust field identifiers,
/// which must remain byte-identical.
#[test]
fn replication_target_usage_rmp_round_trip() {
let original = ReplicationTargetUsage {
pending_size: 100,
replicated_size: 2_000,
failed_size: 50,
failed_count: 3,
pending_count: 7,
missed_threshold_size: 11,
after_threshold_size: 22,
missed_threshold_count: 1,
after_threshold_count: 2,
replicated_count: 99,
};
let buf = rmp_serde::to_vec_named(&original).expect("encode ReplicationTargetUsage to msgpack");
let decoded: ReplicationTargetUsage = rmp_serde::from_slice(&buf).expect("decode ReplicationTargetUsage from msgpack");
assert_eq!(original, decoded, "round-trip through rmp must preserve every field");
// Also verify that encoding as an unnamed sequence and then decoding
// with named fields produces the correct mapping (this catches reordering).
let named_buf = rmp_serde::to_vec_named(&original).expect("re-encode for field-name pinning");
// Spot-check that known field names appear in the named encoding.
let named_str = String::from_utf8_lossy(&named_buf);
assert!(named_str.contains("pending_size"), "field 'pending_size' must survive the rename");
assert!(named_str.contains("replicated_size"), "field 'replicated_size' must survive the rename");
assert!(
named_str.contains("missed_threshold_size"),
"field 'missed_threshold_size' must survive the rename"
);
assert!(
named_str.contains("after_threshold_count"),
"field 'after_threshold_count' must survive the rename"
);
}
#[test]
fn checked_merge_rejects_noncanonical_histograms_without_mutation() {
let mut entry = DataUsageEntry {
crates/e2e_test/AGENTS.md
-1
View File
@@ -28,4 +28,3 @@ follow.
## Suggested Validation
- `cargo test --package e2e_test`
- Full gate before commit: `make pre-commit`
crates/e2e_test/Cargo.toml
-1
View File
@@ -96,7 +96,6 @@ tokio-stream = { workspace = true }
rustfs-madmin.workspace = true
rustfs-filemeta.workspace = true
bytes = { workspace = true, features = ["serde"] }
serial_test = { workspace = true }
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-config = { workspace = true }
crates/e2e_test/README.md
+26 -21
View File
@@ -48,16 +48,14 @@ cargo nextest run --profile e2e-smoke -p e2e_test
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
# Protocols suite — fixed ports, MUST be single-threaded, gated by build features
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
cargo test -p e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
```
The protocols suite has its own contract (fixed bind ports 9022–9301,
`--test-threads=1`, feature-gated scheduling) documented in
single-worker execution, feature-gated scheduling) documented in
[`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
selects which features the spawned binary is built with; leave it unset to run
every protocol entry.
every protocol entry. Use the exact profile command under
[Troubleshooting](#troubleshooting) for CI-equivalent execution.
### `#[ignore]` semantics
@@ -159,27 +157,26 @@ construction (random port + isolated temp dir) and need no serialization.
## CI map
`e2e_test` is **excluded** from the main `cargo nextest run --profile ci --all`
pass ([`.github/workflows/ci.yml`](../../.github/workflows/ci.yml) line 158,
`--exclude e2e_test`) — the whole crate is too slow to gate every PR. Subsets
join CI through the nextest profile system only (never as ad-hoc jobs):
pass (`--exclude e2e_test`) — the whole crate is too slow to gate every PR.
Subsets join CI through nextest profiles; the fixed-port protocol suite uses
the same profile for membership and execution with one nightly worker.
| Suite | Runs where | Status |
| --- | --- | --- |
| Smoke subset (`e2e-smoke` profile) | `e2e-tests` job, every PR | **Active** (backlog#1149 ci-4) |
| Full single-node suite (`e2e-full` profile) | `e2e-full` job, merge queue + main | **Active** (backlog#1149 ci-5) |
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
| KMS suite | — | Not in CI yet (backlog#1149 ci-5) |
| Protocols (FTPS/WebDAV/SFTP) | — | Not in CI yet (backlog#1149 ci-7) |
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
| Replication (slow + dual-node) | `e2e-repl-nightly` profile, scheduled workflow | **Active** (backlog#1147 repl-1) |
| `reliant/*` (pre-started server) | — | Manual only |
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
| `reliant/*` | 19 tests in PR smoke; remaining default tests in `e2e-full` | **Active** except `#[ignore]` |
Links: [`ci.yml`](../../.github/workflows/ci.yml) `e2e-tests` (line 347),
`test-ilm-integration-serial` (line 196). The `e2e-smoke` `default-filter` in
[`.config/nextest.toml`](../../.config/nextest.toml) is the **single wiring
mechanism** — extend that filter (or add a sibling profile) to admit more
tests; do not add e2e jobs to `ci.yml`. repl-1 / ilm-3 are landing in parallel
and may add lanes; keep the table above easy to extend.
The profile filters in [`.config/nextest.toml`](../../.config/nextest.toml) are
the wiring source of truth. Committed test-ID digests under
`.config/e2e-*-selection.txt` make every membership change explicit.
## Troubleshooting
@@ -188,9 +185,15 @@ and may add lanes; keep the table above easy to extend.
```bash
# Smoke (e2e-tests job) — includes the 20 fast replication tests
cargo nextest run --profile e2e-smoke -p e2e_test
# Replication nightly lane (16 slow + dual-node tests; install awscurl for the
# STS dual-node test, else it skips gracefully)
# Full single-node merge/main lane
cargo nextest run --profile e2e-full -p e2e_test
# Cluster fault nightly lane
cargo nextest run --profile e2e-nightly -p e2e_test
# Replication nightly lane; install awscurl so STS paths do not skip
cargo nextest run --profile e2e-repl-nightly -p e2e_test
# Fixed-port protocol nightly lane
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
# ILM serial lane
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
@@ -273,4 +276,6 @@ current subset is.
`docs/testing/e2e-suite-inventory.md` records the per-module test counts as
listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
moving e2e tests so acceptance numbers in the test-strategy issues
(backlog#1147–#1155) stay auditable.
(backlog#1147–#1155) stay auditable. When a profile membership change is
intentional, review its JSON listing before updating the matching
`.config/e2e-*-selection.txt` test-ID digest.
crates/e2e_test/src/common.rs
+151
View File
@@ -30,6 +30,7 @@ use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serde_json;
use std::ffi::OsStr;
use std::fs as stdfs;
use std::io::ErrorKind;
@@ -1583,6 +1584,156 @@ impl Drop for RustFSTestClusterEnvironment {
}
}
/// Send a SigV4-signed HTTP request and return the raw `reqwest::Response`.
///
/// Unlike [`signed_s3_request`], this variant accepts `body: Option<Vec<u8>>`
/// (binary-safe) and reorders parameters so that `access_key`/`secret_key`
/// appear before the body — matching the convention used by the replication
/// extension and object-lambda e2e suites.
pub(crate) async fn signed_request(
method: http::Method,
url: &str,
access_key: &str,
secret_key: &str,
body: Option<Vec<u8>>,
content_type: Option<&str>,
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let mut request = http::Request::builder().method(method.clone()).uri(uri);
request = request.header(HOST, authority);
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if let Some(content_type) = content_type {
request = request.header(CONTENT_TYPE, content_type);
}
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
let client = local_http_client();
let mut request_builder = client.request(reqwest_method, url);
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if let Some(body) = body {
request_builder = request_builder.body(body);
}
Ok(request_builder.send().await?)
}
/// Like [`signed_request`], but uses a caller-supplied `reqwest::Client`
/// instead of the shared [`local_http_client`].
pub(crate) async fn signed_request_with_client(
client: &reqwest::Client,
method: http::Method,
url: &str,
access_key: &str,
secret_key: &str,
body: Option<Vec<u8>>,
content_type: Option<&str>,
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let mut request = http::Request::builder().method(method.clone()).uri(uri);
request = request.header(HOST, authority);
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if let Some(content_type) = content_type {
request = request.header(CONTENT_TYPE, content_type);
}
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
let mut request_builder = client.request(reqwest_method, url);
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if let Some(body) = body {
request_builder = request_builder.body(body);
}
Ok(request_builder.send().await?)
}
/// Like [`signed_request`], but includes a `session_token` in the
/// `x-amz-security-token` header and passes it to the SigV4 signer.
pub(crate) async fn signed_request_with_session_token(
method: http::Method,
url: &str,
access_key: &str,
secret_key: &str,
session_token: &str,
body: Option<Vec<u8>>,
content_type: Option<&str>,
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let mut request = http::Request::builder().method(method.clone()).uri(uri);
request = request.header(HOST, authority);
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if !session_token.is_empty() {
request = request.header("x-amz-security-token", session_token);
}
if let Some(content_type) = content_type {
request = request.header(CONTENT_TYPE, content_type);
}
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
let signed = sign_v4(
request.body(Body::empty())?,
content_len,
access_key,
secret_key,
session_token,
"us-east-1",
);
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
let client = local_http_client();
let mut request_builder = client.request(reqwest_method, url);
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if let Some(body) = body {
request_builder = request_builder.body(body);
}
Ok(request_builder.send().await?)
}
/// Create a new user via the admin API.
pub(crate) async fn admin_create_user(
env: &RustFSTestEnvironment,
username: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
let body = serde_json::json!({
"secretKey": secret_key,
"status": "enabled"
});
let response = signed_request(
http::Method::PUT,
&url,
&env.access_key,
&env.secret_key,
Some(body.to_string().into_bytes()),
Some("application/json"),
)
.await?;
if response.status() != reqwest::StatusCode::OK {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(format!("create user failed: {status} {body}").into());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
crates/e2e_test/src/degraded_read_eof_regression_test.rs
-4
View File
@@ -55,7 +55,6 @@ mod tests {
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::error::Error;
use tokio::time::{Duration, timeout};
@@ -269,7 +268,6 @@ mod tests {
/// stripes) and a multipart object (3 parts × 5 MiB) must GET back as a
/// full, byte-identical body with the correct Content-Length. No early EOF.
#[tokio::test]
#[serial]
async fn degraded_read_large_objects_with_one_disk_offline_return_full_body() -> TestResult {
init_logging();
info!("dist-13 (a): large-object degraded read with one of four disks offline");
@@ -335,7 +333,6 @@ mod tests {
/// mid-stream — the exact window the fixes had to reconstruct through rather
/// than truncate.
#[tokio::test]
#[serial]
async fn degraded_read_reconstructs_through_midstream_bitrot_within_quorum() -> TestResult {
init_logging();
info!("dist-13 (b): mid-stream bitrot within quorum must reconstruct a full body");
@@ -393,7 +390,6 @@ mod tests {
/// Content-Length. `get_checked` panics on that forbidden outcome, so this
/// test fails loudly if the truncation bug ever returns.
#[tokio::test]
#[serial]
async fn beyond_quorum_degraded_read_never_silently_truncates() -> TestResult {
init_logging();
info!("dist-13 (c): beyond-quorum degraded read must fail, never 200+truncated");
crates/e2e_test/src/get_stream_failure_observability_test.rs
-2
View File
@@ -51,7 +51,6 @@ mod tests {
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use serial_test::serial;
use std::error::Error;
use tokio::time::{Duration, timeout};
use tracing::info;
@@ -129,7 +128,6 @@ mod tests {
/// the body — and assert the server log names the object, at the log level a
/// default deployment actually runs with.
#[tokio::test]
#[serial]
async fn midstream_get_failure_is_logged_with_the_object_at_default_log_level() -> TestResult {
init_logging();
info!("rustfs#4784: a mid-stream GET failure must name its object in the source log");
crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs
+18 -4
View File
@@ -380,10 +380,24 @@ mod tests {
cluster.start_node(1).await?;
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
assert!(
!status_body.contains("MissingContentLength"),
"background heal status should not fail without an explicit Content-Length: {status_body}"
let mut recovered = serde_json::Value::Null;
for _ in 0..60 {
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
assert!(
!status_body.contains("MissingContentLength"),
"background heal status should not fail without an explicit Content-Length: {status_body}"
);
recovered = serde_json::from_str(&status_body)
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
break;
}
sleep(Duration::from_secs(1)).await;
}
assert_eq!(
recovered["clusterStatusComplete"],
serde_json::Value::Bool(true),
"cluster heal status should recover before root heal starts: {recovered}"
);
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
crates/e2e_test/src/inline_fast_path_cluster_test.rs
-15
View File
@@ -46,7 +46,6 @@ use prost::Message;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serial_test::serial;
use std::collections::BTreeMap;
use std::convert::Infallible;
use std::error::Error;
@@ -1695,7 +1694,6 @@ fn assert_storage_layout(
}
#[tokio::test]
#[serial]
async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
init_logging();
@@ -1767,7 +1765,6 @@ async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
}
#[tokio::test]
#[serial]
async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
init_logging();
@@ -1805,7 +1802,6 @@ async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
}
#[tokio::test]
#[serial]
async fn four_node_inline_fallback_controls() -> TestResult {
init_logging();
@@ -1870,7 +1866,6 @@ async fn four_node_inline_fallback_controls() -> TestResult {
}
#[tokio::test]
#[serial]
async fn four_node_compressed_inline_fallback() -> TestResult {
init_logging();
@@ -1905,7 +1900,6 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
/// Reverting the multipart compression fix must fail this test.
#[tokio::test]
#[serial]
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
init_logging();
@@ -1952,7 +1946,6 @@ async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
/// read costs on the order of the covering part's block size against a ~5 MiB
/// object.
#[tokio::test]
#[serial]
async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult {
init_logging();
@@ -2019,7 +2012,6 @@ async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestRe
}
#[tokio::test]
#[serial]
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
init_logging();
@@ -2123,7 +2115,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
}
#[tokio::test]
#[serial]
async fn four_node_add_tier_converges() -> TestResult {
init_logging();
@@ -2142,7 +2133,6 @@ async fn four_node_add_tier_converges() -> TestResult {
}
#[tokio::test]
#[serial]
async fn four_node_add_tier_converges_after_offline_node_restart_without_second_mutation() -> TestResult {
init_logging();
@@ -2164,7 +2154,6 @@ async fn four_node_add_tier_converges_after_offline_node_restart_without_second_
}
#[tokio::test]
#[serial]
async fn four_node_manual_transition_job_status_survives_node_restart() -> TestResult {
init_logging();
@@ -2239,7 +2228,6 @@ async fn four_node_manual_transition_job_status_survives_node_restart() -> TestR
}
#[tokio::test]
#[serial]
async fn four_node_manual_transition_distributed_admission_conflict_reports_status_and_backpressure() -> TestResult {
init_logging();
@@ -2381,7 +2369,6 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
}
#[tokio::test]
#[serial]
#[ignore = "manual #1508 evidence harness: starts a 4-node cluster, a remote tier, and an in-flight transition job"]
async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> TestResult {
init_logging();
@@ -2486,7 +2473,6 @@ async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> Tes
}
#[tokio::test]
#[serial]
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_transition() -> TestResult {
init_logging();
@@ -2598,7 +2584,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
}
#[tokio::test]
#[serial]
async fn four_node_transitioned_inline_fallback() -> TestResult {
init_logging();
crates/e2e_test/src/kms/bucket_default_encryption_test.rs
+5 -5
View File
@@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -159,7 +159,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -278,7 +278,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -475,7 +475,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -570,7 +570,7 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/common.rs
+138 -1
View File
@@ -40,7 +40,7 @@ use std::time::Duration;
use tokio::fs;
use tokio::net::TcpStream;
use tokio::time::sleep;
use tracing::{debug, error, info};
use tracing::{debug, error, info, warn};
// KMS-specific constants
pub const TEST_BUCKET: &str = "kms-test-bucket";
@@ -177,6 +177,136 @@ pub async fn get_kms_status(
Ok(status)
}
/// Poll the KMS status endpoint until the backend reports ready or the timeout
/// expires. Replaces hard-coded `sleep(Duration::from_secs(3))` startup waits
/// with an active readiness probe so tests start as soon as KMS is usable
/// (typically < 1 s) instead of always waiting the full 3 s.
///
/// Uses exponential back-off starting at 200 ms (doubling each attempt, capped
/// at 1 s) up to a total wall-clock budget of 5 s.
pub async fn wait_for_kms_ready(
base_url: &str,
access_key: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await
}
async fn wait_for_kms_ready_with_timeout(
base_url: &str,
access_key: &str,
secret_key: &str,
total_deadline: Duration,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let start = tokio::time::Instant::now();
let deadline = start + total_deadline;
let mut backoff = Duration::from_millis(200);
let max_backoff = Duration::from_secs(1);
loop {
match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await {
Ok(Ok(status)) => {
let backend_status = serde_json::from_str::<serde_json::Value>(&status)
.ok()
.and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned));
if backend_status.as_deref() == Some("healthy") {
info!("KMS is ready (status: {})", status);
return Ok(());
}
warn!(
backend_status = backend_status.as_deref().unwrap_or("missing"),
elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
"KMS not ready yet, retrying…"
);
}
Ok(Err(e)) => {
let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…");
}
Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()),
}
let now = tokio::time::Instant::now();
if now >= deadline {
return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into());
}
sleep((now + backoff).min(deadline) - now).await;
backoff = (backoff * 2).min(max_backoff);
}
}
#[cfg(test)]
mod readiness_tests {
use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
#[tokio::test]
async fn kms_readiness_retries_http_success_until_backend_is_healthy() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
let address = listener.local_addr().expect("read readiness test server address");
let requests = Arc::new(AtomicUsize::new(0));
let server_requests = Arc::clone(&requests);
let server = tokio::spawn(async move {
for backend_status in ["error", "healthy"] {
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
let mut request = Vec::new();
let mut chunk = [0_u8; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let read = socket.read(&mut chunk).await.expect("read readiness request");
if read == 0 {
break;
}
request.extend_from_slice(&chunk[..read]);
}
server_requests.fetch_add(1, Ordering::SeqCst);
let body = format!(r#"{{"backend_status":"{backend_status}"}}"#);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
socket.write_all(response.as_bytes()).await.expect("write readiness response");
}
});
wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key")
.await
.expect("KMS should become ready after the healthy response");
let observed_requests = requests.load(Ordering::SeqCst);
server.abort();
assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried");
}
#[tokio::test]
async fn kms_readiness_deadline_covers_a_stalled_status_request() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
let address = listener.local_addr().expect("read readiness test server address");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
let mut request = [0_u8; 1024];
let _ = socket.read(&mut request).await.expect("read readiness request");
std::future::pending::<()>().await;
});
let result = tokio::time::timeout(
Duration::from_secs(1),
wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)),
)
.await
.expect("readiness helper must enforce its own deadline");
server.abort();
assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline");
}
}
/// Create a default KMS key for testing and return the created key ID
pub async fn create_default_key(
base_url: &str,
@@ -861,6 +991,13 @@ impl LocalKMSTestEnvironment {
Ok(default_key_id.to_string())
}
/// Poll the KMS status endpoint until the backend reports ready.
///
/// Prefer this over a fixed `sleep` after calling `start_rustfs_for_local_kms`.
pub async fn wait_for_kms_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
wait_for_kms_ready(&self.base_env.url, &self.base_env.access_key, &self.base_env.secret_key).await
}
/// Configure Local KMS backend with a predefined default key
pub async fn configure_local_kms(&self) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
// Use a fixed, predictable default key ID
crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs
+3 -3
View File
@@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
)
.await
.expect("failed to start RustFS with local KMS");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
let client = kms_env.base_env.create_s3_client();
// Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
@@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
)
.await
.expect("failed to start RustFS with local KMS");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
let client = kms_env.base_env.create_s3_client();
// Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
@@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta
)
.await
.expect("failed to start RustFS with local KMS");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
let client = kms_env.base_env.create_s3_client();
let bucket = "copy-object-self-copy-bucket-default-sse-test";
crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs
+1 -1
View File
@@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
)
.await
.expect("failed to start RustFS with local KMS");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
let client = kms_env.base_env.create_s3_client();
let bucket = "copy-object-version-restore-sse-test";
crates/e2e_test/src/kms/encryption_metadata_test.rs
+3 -3
View File
@@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -147,7 +147,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/kms_comprehensive_test.rs
+5 -6
View File
@@ -24,7 +24,6 @@ use super::common::{
test_sse_kms_encryption, test_sse_s3_encryption,
};
use crate::common::{TEST_BUCKET, init_logging};
use tokio::time::{Duration, sleep};
use tracing::info;
/// Comprehensive test: Full KMS workflow with all encryption types
@@ -35,7 +34,7 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -103,7 +102,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -137,7 +136,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -208,7 +207,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -253,7 +252,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/kms_edge_cases_test.rs
+6 -6
View File
@@ -44,7 +44,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -117,7 +117,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -209,7 +209,7 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box<dyn std::err
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -284,7 +284,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -371,7 +371,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = Arc::new(kms_env.base_env.create_s3_client());
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -478,7 +478,7 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/kms_fault_recovery_test.rs
+4 -4
View File
@@ -37,7 +37,7 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -127,7 +127,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -218,7 +218,7 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -401,7 +401,7 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/kms_local_test.rs
+4 -4
View File
@@ -46,7 +46,7 @@ async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
.expect("Failed to start RustFS with Local KMS");
// Wait a moment for RustFS to fully start up and initialize KMS
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
@@ -127,7 +127,7 @@ async fn test_local_kms_key_isolation() {
.expect("Failed to start RustFS with Local KMS");
// Wait a moment for RustFS to fully start up and initialize KMS
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
@@ -227,7 +227,7 @@ async fn test_local_kms_large_file() {
.expect("Failed to start RustFS with Local KMS");
// Wait a moment for RustFS to fully start up and initialize KMS
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
@@ -309,7 +309,7 @@ async fn test_local_kms_multipart_upload() {
.expect("Failed to start RustFS with Local KMS");
// Wait for KMS initialization
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
crates/e2e_test/src/kms/kms_vault_test.rs
+2 -9
View File
@@ -19,8 +19,6 @@
//! multipart upload behaviour.
use crate::common::{TEST_BUCKET, init_logging};
use serial_test::serial;
use tokio::time::{Duration, sleep};
use tracing::{error, info};
use super::common::{
@@ -46,8 +44,8 @@ impl VaultKmsTestContext {
start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
// Allow Vault to finish initialising token auth and transit engine.
sleep(Duration::from_secs(2)).await;
// Wait for KMS to finish initialising.
super::common::wait_for_kms_ready(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
Ok(Self { env })
}
@@ -62,7 +60,6 @@ impl VaultKmsTestContext {
}
#[tokio::test]
#[serial]
async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
@@ -118,7 +115,6 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
}
#[tokio::test]
#[serial]
async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
@@ -205,7 +201,6 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
}
#[tokio::test]
#[serial]
async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
@@ -270,7 +265,6 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
}
#[tokio::test]
#[serial]
async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
@@ -301,7 +295,6 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
}
#[tokio::test]
#[serial]
async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
crates/e2e_test/src/kms/mod.rs
-3
View File
@@ -39,9 +39,6 @@ mod kms_edge_cases_test;
#[cfg(test)]
mod kms_fault_recovery_test;
#[cfg(test)]
mod test_runner;
#[cfg(test)]
mod bucket_default_encryption_test;
crates/e2e_test/src/kms/multipart_encryption_test.rs
+5 -5
View File
@@ -33,7 +33,7 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -89,7 +89,7 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -310,7 +310,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -435,7 +435,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/test_runner.rs
-499
View File
@@ -1,499 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
#![allow(dead_code)]
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Unified KMS test suite runner
//!
//! This module provides a unified interface for running KMS tests with categorization,
//! filtering, and comprehensive reporting capabilities.
use crate::common::init_logging;
use std::time::Instant;
use tokio::time::{Duration, sleep};
use tracing::{debug, error, info, warn};
/// Test category for organization and filtering
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TestCategory {
CoreFunctionality,
MultipartEncryption,
EdgeCases,
FaultRecovery,
Comprehensive,
Performance,
}
impl TestCategory {
pub fn as_str(&self) -> &'static str {
match self {
TestCategory::CoreFunctionality => "core-functionality",
TestCategory::MultipartEncryption => "multipart-encryption",
TestCategory::EdgeCases => "edge-cases",
TestCategory::FaultRecovery => "fault-recovery",
TestCategory::Comprehensive => "comprehensive",
TestCategory::Performance => "performance",
}
}
}
/// Test definition with metadata
#[derive(Debug, Clone)]
pub struct TestDefinition {
pub name: String,
pub description: String,
pub category: TestCategory,
pub estimated_duration: Duration,
pub is_critical: bool,
}
impl TestDefinition {
pub fn new(
name: impl Into<String>,
description: impl Into<String>,
category: TestCategory,
estimated_duration: Duration,
is_critical: bool,
) -> Self {
Self {
name: name.into(),
description: description.into(),
category,
estimated_duration,
is_critical,
}
}
}
/// Test execution result
#[derive(Debug, Clone)]
pub struct TestResult {
pub test_name: String,
pub category: TestCategory,
pub success: bool,
pub duration: Duration,
pub error_message: Option<String>,
}
impl TestResult {
pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
Self {
test_name,
category,
success: true,
duration,
error_message: None,
}
}
pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
Self {
test_name,
category,
success: false,
duration,
error_message: Some(error),
}
}
}
/// Comprehensive test suite configuration
#[derive(Debug, Clone)]
pub struct TestSuiteConfig {
pub categories: Vec<TestCategory>,
pub include_critical_only: bool,
pub max_duration: Option<Duration>,
pub parallel_execution: bool,
}
impl Default for TestSuiteConfig {
fn default() -> Self {
Self {
categories: vec![
TestCategory::CoreFunctionality,
TestCategory::MultipartEncryption,
TestCategory::EdgeCases,
TestCategory::FaultRecovery,
TestCategory::Comprehensive,
],
include_critical_only: false,
max_duration: None,
parallel_execution: false,
}
}
}
/// Unified KMS test suite runner
pub struct KMSTestSuite {
tests: Vec<TestDefinition>,
config: TestSuiteConfig,
}
impl KMSTestSuite {
/// Create a new test suite with default configuration
pub fn new() -> Self {
let tests = vec![
// Core Functionality Tests
TestDefinition::new(
"test_local_kms_end_to_end",
"End-to-end KMS test with all encryption types",
TestCategory::CoreFunctionality,
Duration::from_secs(60),
true,
),
TestDefinition::new(
"test_local_kms_key_isolation",
"Test KMS key isolation and security",
TestCategory::CoreFunctionality,
Duration::from_secs(45),
true,
),
// Multipart Encryption Tests
TestDefinition::new(
"test_local_kms_multipart_upload",
"Test large file multipart upload with encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(120),
true,
),
TestDefinition::new(
"test_step1_basic_single_file_encryption",
"Basic single file encryption test",
TestCategory::MultipartEncryption,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_step2_basic_multipart_upload_without_encryption",
"Basic multipart upload without encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_step3_multipart_upload_with_sse_s3",
"Multipart upload with SSE-S3 encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(60),
true,
),
TestDefinition::new(
"test_step4_large_multipart_upload_with_encryption",
"Large file multipart upload with encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(90),
false,
),
TestDefinition::new(
"test_step5_all_encryption_types_multipart",
"All encryption types multipart test",
TestCategory::MultipartEncryption,
Duration::from_secs(120),
true,
),
// Edge Cases Tests
TestDefinition::new(
"test_kms_zero_byte_file_encryption",
"Test encryption of zero-byte files",
TestCategory::EdgeCases,
Duration::from_secs(20),
false,
),
TestDefinition::new(
"test_kms_single_byte_file_encryption",
"Test encryption of single-byte files",
TestCategory::EdgeCases,
Duration::from_secs(20),
false,
),
TestDefinition::new(
"test_kms_multipart_boundary_conditions",
"Test multipart upload boundary conditions",
TestCategory::EdgeCases,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_kms_invalid_key_scenarios",
"Test invalid key scenarios",
TestCategory::EdgeCases,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_kms_concurrent_encryption",
"Test concurrent encryption operations",
TestCategory::EdgeCases,
Duration::from_secs(60),
false,
),
TestDefinition::new(
"test_kms_key_validation_security",
"Test key validation security",
TestCategory::EdgeCases,
Duration::from_secs(30),
false,
),
// Fault Recovery Tests
TestDefinition::new(
"test_kms_key_directory_unavailable",
"Test KMS when key directory is unavailable",
TestCategory::FaultRecovery,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_kms_corrupted_key_files",
"Test KMS with corrupted key files",
TestCategory::FaultRecovery,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_kms_multipart_upload_interruption",
"Test multipart upload interruption recovery",
TestCategory::FaultRecovery,
Duration::from_secs(60),
false,
),
TestDefinition::new(
"test_kms_resource_constraints",
"Test KMS under resource constraints",
TestCategory::FaultRecovery,
Duration::from_secs(90),
false,
),
// Comprehensive Tests
TestDefinition::new(
"test_comprehensive_kms_full_workflow",
"Full KMS workflow comprehensive test",
TestCategory::Comprehensive,
Duration::from_secs(300),
true,
),
TestDefinition::new(
"test_comprehensive_stress_test",
"KMS stress test with large datasets",
TestCategory::Comprehensive,
Duration::from_secs(400),
false,
),
TestDefinition::new(
"test_comprehensive_key_isolation",
"Comprehensive key isolation test",
TestCategory::Comprehensive,
Duration::from_secs(180),
false,
),
TestDefinition::new(
"test_comprehensive_concurrent_operations",
"Comprehensive concurrent operations test",
TestCategory::Comprehensive,
Duration::from_secs(240),
false,
),
TestDefinition::new(
"test_comprehensive_performance_benchmark",
"KMS performance benchmark test",
TestCategory::Comprehensive,
Duration::from_secs(360),
false,
),
];
Self {
tests,
config: TestSuiteConfig::default(),
}
}
/// Configure the test suite
pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
self.config = config;
self
}
/// Filter tests based on category
pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
self.tests.iter().filter(|test| &test.category == category).collect()
}
/// Filter tests based on criticality
pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
self.tests.iter().filter(|test| test.is_critical).collect()
}
/// Get test summary by category
pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
let mut summary = std::collections::HashMap::new();
for test in &self.tests {
summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
}
summary
}
/// Run the complete test suite
pub async fn run_test_suite(&self) -> Vec<TestResult> {
init_logging();
info!("🚀 Starting unified KMS test suite");
let start_time = Instant::now();
let mut results = Vec::new();
// Filter tests based on configuration
let tests_to_run: Vec<&TestDefinition> = self
.tests
.iter()
.filter(|test| self.config.categories.contains(&test.category))
.filter(|test| !self.config.include_critical_only || test.is_critical)
.collect();
info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
for (i, test) in tests_to_run.iter().enumerate() {
info!(" {}. {} ({})", i + 1, test.name, test.category.as_str());
}
// Execute tests
for (i, test_def) in tests_to_run.iter().enumerate() {
info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
info!(" 📝 Description: {}", test_def.description);
info!(" 🏷️ Category: {}", test_def.category.as_str());
info!(" ⏱️ Estimated duration: {:?}", test_def.estimated_duration);
let test_start = Instant::now();
let result = self.run_single_test(test_def).await;
let test_duration = test_start.elapsed();
match result {
Ok(_) => {
info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
}
Err(e) => {
error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
results.push(TestResult::failure(
test_def.name.clone(),
test_def.category.clone(),
test_duration,
e.to_string(),
));
}
}
// Add delay between tests to avoid resource conflicts
if i < tests_to_run.len() - 1 {
debug!("⏸️ Waiting two seconds before the next test...");
sleep(Duration::from_secs(2)).await;
}
}
let total_duration = start_time.elapsed();
self.print_test_summary(&results, total_duration);
results
}
/// Run a single test by dispatching to the appropriate test function
async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// This is a placeholder for test dispatch logic
// In a real implementation, this would dispatch to actual test functions
warn!("⚠️ Test '{}' is not implemented in the unified runner; skipping", test_def.name);
Ok(())
}
/// Print comprehensive test summary
fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
info!("📊 KMS test suite summary");
info!("⏱️ Total duration: {:.2} seconds", total_duration.as_secs_f64());
info!("📈 Total tests: {}", results.len());
let passed = results.iter().filter(|r| r.success).count();
let failed = results.iter().filter(|r| !r.success).count();
info!("✅ Passed: {}", passed);
info!("❌ Failed: {}", failed);
info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
// Summary by category
let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
for result in results {
let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
*total += 1;
if result.success {
*passed_count += 1;
}
}
info!("📊 Category summary:");
for (category, (total, passed_count)) in category_summary {
info!(
" 🏷️ {}: {}/{} ({:.1}%)",
category.as_str(),
passed_count,
total,
(passed_count as f64 / total as f64) * 100.0
);
}
// List failed tests
if failed > 0 {
warn!("❌ Failing tests:");
for result in results.iter().filter(|r| !r.success) {
warn!(" - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
}
}
}
}
/// Quick test suite for critical tests only
#[tokio::test]
async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let config = TestSuiteConfig {
categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
include_critical_only: true,
max_duration: Some(Duration::from_secs(600)), // 10 minutes max
parallel_execution: false,
};
let suite = KMSTestSuite::new().with_config(config);
let results = suite.run_test_suite().await;
let failed_count = results.iter().filter(|r| !r.success).count();
if failed_count > 0 {
return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
}
info!("✅ All critical tests passed");
Ok(())
}
/// Full comprehensive test suite
#[tokio::test]
async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let suite = KMSTestSuite::new();
let results = suite.run_test_suite().await;
let total_tests = results.len();
let failed_count = results.iter().filter(|r| !r.success).count();
let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
info!("📊 Full suite success rate: {:.1}%", success_rate);
// Allow up to 10% failure rate for non-critical tests
if success_rate < 90.0 {
return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
}
info!("✅ Full test suite succeeded");
Ok(())
}
crates/e2e_test/src/object_lambda_test.rs
+27 -48
View File
@@ -12,12 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client};
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client, signed_request};
use aws_sdk_s3::primitives::ByteStream;
use http::header::{CONTENT_TYPE, HOST};
use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::{pre_sign_v4, sign_v4};
use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
use rustfs_signer::pre_sign_v4;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use s3s::Body;
use std::collections::HashMap;
@@ -227,39 +227,6 @@ async fn presigned_get_request(
Ok(local_http_client().get(signed.uri().to_string()).send().await?)
}
async fn signed_request(
method: http::Method,
url: &str,
access_key: &str,
secret_key: &str,
body: Option<Vec<u8>>,
content_type: Option<&str>,
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let mut request = http::Request::builder().method(method.clone()).uri(uri);
request = request.header(HOST, authority);
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if let Some(content_type) = content_type {
request = request.header(CONTENT_TYPE, content_type);
}
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
let client = local_http_client();
let mut request_builder = client.request(reqwest_method, url);
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if let Some(body) = body {
request_builder = request_builder.body(body);
}
Ok(request_builder.send().await?)
}
async fn configure_webhook_target(
env: &RustFSTestEnvironment,
target_name: &str,
@@ -1010,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
let bucket = "object-lambda-e2e-disabled-target";
let key = "input.txt";
@@ -1026,17 +994,24 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
.send()
.await?;
configure_webhook_target_with_key_values(
&env,
"transformer",
vec![
("endpoint", "http://127.0.0.1:9/transform".to_string()),
("auth_token", "secret-token".to_string()),
("enable", "off".to_string()),
],
let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
tokio::fs::create_dir_all(&queue_dir).await?;
let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
let directive = format!(
"notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
);
let disable_response = signed_request(
http::Method::PUT,
&config_url,
&env.access_key,
&env.secret_key,
Some(directive.into_bytes()),
Some("text/plain"),
)
.await?;
wait_for_target_visibility(&env, "transformer").await?;
let disable_status = disable_response.status();
let disable_body = disable_response.text().await?;
assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");
let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
@@ -1055,7 +1030,8 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
let bucket = "object-lambda-e2e-invalid-endpoint";
@@ -1098,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
let response = send_configure_webhook_target_request(
&env,
@@ -1207,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
init_logging();
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
cluster.set_env(ENV_NOTIFY_ENABLE, "true");
cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
cluster.start().await?;
let bucket = "listen-notification-cluster";
crates/e2e_test/src/policy/policy_variables_test.rs
-7
View File
@@ -17,7 +17,6 @@
use crate::common::{awscurl_delete, awscurl_put, init_logging};
use crate::policy::test_env::PolicyTestEnvironment;
use aws_sdk_s3::primitives::ByteStream;
use serial_test::serial;
use tracing::info;
/// Helper function to create a regular user with given credentials
@@ -122,7 +121,6 @@ async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, po
/// Test AWS policy variables with single-value scenarios
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_single_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_single_value_impl().await
@@ -275,7 +273,6 @@ pub async fn test_aws_policy_variables_single_value_impl_with_env(
/// Test AWS policy variables with multi-value scenarios
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_multi_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_multi_value_impl().await
@@ -401,7 +398,6 @@ pub async fn test_aws_policy_variables_multi_value_impl_with_env(
/// Test AWS policy variables with variable concatenation
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_concatenation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_concatenation_impl().await
@@ -491,7 +487,6 @@ pub async fn test_aws_policy_variables_concatenation_impl_with_env(
/// Test AWS policy variables with nested scenarios
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_nested() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_nested_impl().await
@@ -509,7 +504,6 @@ pub async fn test_aws_policy_variables_nested_impl() -> Result<(), Box<dyn std::
/// Test AWS policy variables with STS temporary credentials
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_sts() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_sts_impl().await
@@ -705,7 +699,6 @@ pub async fn test_aws_policy_variables_sts_impl_with_env(
/// Test AWS policy variables with deny scenarios
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_deny() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_deny_impl().await
crates/e2e_test/src/policy/test_runner.rs
-2
View File
@@ -14,7 +14,6 @@
use crate::common::init_logging;
use crate::policy::test_env::PolicyTestEnvironment;
use serial_test::serial;
use std::time::Instant;
use tokio::time::{Duration, sleep};
use tracing::{error, info};
@@ -213,7 +212,6 @@ impl PolicyTestSuite {
/// Test suite
#[tokio::test]
#[serial]
#[ignore = "Connects to existing rustfs server"]
async fn test_policy_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let config = TestSuiteConfig {
crates/e2e_test/src/protocols/README.md
+7 -1
View File
@@ -11,10 +11,17 @@ test process directly.
## Running Tests
Use the canonical CI-equivalent protocol command in the parent
[`e2e_test` README](../../README.md#troubleshooting).
For targeted debugging of the core suite only:
```bash
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
```
This targeted command does not cover the full `e2e-protocols` profile.
`RUSTFS_BUILD_FEATURES` controls which features the test rustfs binary is
built with. When this variable is set, the protocol test runner schedules
only entries whose protocol is present in the requested feature list. Leave
@@ -133,4 +140,3 @@ property without consulting any external doc.
Bind ports 9023 (SFTP) and 9100 (S3). Spawns rustfs with
`RUSTFS_SFTP_IDLE_TIMEOUT=5`, sleeps 10 s past the timeout, then issues an
SFTP request and asserts the server has closed the session.
crates/e2e_test/src/protocols/webdav_core.rs
-2
View File
@@ -41,7 +41,6 @@ use reqwest::Client;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serial_test::serial;
use tokio::process::Command;
use tracing::info;
@@ -821,7 +820,6 @@ pub async fn test_webdav_core_operations() -> Result<()> {
}
#[tokio::test]
#[serial]
async fn test_webdav_core_operations_direct() -> Result<()> {
test_webdav_core_operations().await
}
crates/e2e_test/src/reliability_disk_fault_test.rs
-5
View File
@@ -27,7 +27,6 @@ mod tests {
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::error::Error;
@@ -157,7 +156,6 @@ mod tests {
/// content, degraded writes must succeed, and everything must still
/// verify after the disk returns.
#[tokio::test]
#[serial]
async fn test_degraded_read_write_with_one_disk_offline() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Reliability: degraded read/write with one of four disks offline");
@@ -210,7 +208,6 @@ mod tests {
/// bytes to a reader: per-shard bitrot checksums reject the bad shard and
/// the object is reconstructed from the remaining shards.
#[tokio::test]
#[serial]
async fn test_bitrot_corrupted_shard_read_returns_correct_data() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Reliability: GET must read through a bitrot-corrupted shard");
@@ -253,7 +250,6 @@ mod tests {
/// heal, and require the replaced disk to be rebuilt and all content to
/// verify against the sha256 manifest.
#[tokio::test]
#[serial]
async fn test_fresh_disk_replacement_heals_after_sigkill_restart() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Reliability: fresh-disk replacement heals after SIGKILL restart");
@@ -327,7 +323,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_versioned_shard_census_selects_each_version_data_dir() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
info!("Reliability: physical shard census selects the requested object version");
crates/e2e_test/src/replacement_privileged_e2e_test.rs
-3
View File
@@ -29,7 +29,6 @@ mod tests {
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
use http::Method;
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::collections::BTreeSet;
use std::error::Error;
@@ -1061,7 +1060,6 @@ mod tests {
/// Linux mount namespaces are per-thread; keep mount setup and process
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
#[tokio::test(flavor = "current_thread")]
#[serial]
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
async fn test_privileged_3x4_auto_replacement_rebuilds_ec8_plus_4_without_admin_heal()
-> Result<(), Box<dyn Error + Send + Sync>> {
@@ -1075,7 +1073,6 @@ mod tests {
/// Linux mount namespaces are per-thread; keep mount setup and process
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
#[tokio::test(flavor = "current_thread")]
#[serial]
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_REPLACEMENT_E2E=1"]
async fn test_privileged_3x4_auto_replacement_rebuilds_ec6_plus_6_without_admin_heal()
-> Result<(), Box<dyn Error + Send + Sync>> {
crates/e2e_test/src/replication_extension_test.rs
+4 -145
View File
@@ -13,8 +13,9 @@
// limitations under the License.
use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
replication_fast_env, rustfs_binary_path,
RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
signed_request_with_session_token,
};
use crate::fake_s3_target::{
FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -35,7 +36,7 @@ use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use bytes::Bytes;
use flate2::read::GzDecoder;
use futures::{Stream, StreamExt};
use http::header::{CONTENT_ENCODING, CONTENT_TYPE, HOST};
use http::header::CONTENT_ENCODING;
use http_body_util::{BodyExt, Full};
use hyper::body::Incoming;
use hyper::server::conn::http1;
@@ -56,9 +57,6 @@ use rustfs_madmin::{
AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use s3s::header::X_AMZ_REPLICATION_STATUS;
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
@@ -387,116 +385,6 @@ struct ReplicationResetStatusTarget {
object: String,
}
async fn signed_request(
method: http::Method,
url: &str,
access_key: &str,
secret_key: &str,
body: Option<Vec<u8>>,
content_type: Option<&str>,
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let mut request = http::Request::builder().method(method.clone()).uri(uri);
request = request.header(HOST, authority);
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if let Some(content_type) = content_type {
request = request.header(CONTENT_TYPE, content_type);
}
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
let client = local_http_client();
let mut request_builder = client.request(reqwest_method, url);
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if let Some(body) = body {
request_builder = request_builder.body(body);
}
Ok(request_builder.send().await?)
}
async fn signed_request_with_client(
client: &reqwest::Client,
method: http::Method,
url: &str,
access_key: &str,
secret_key: &str,
body: Option<Vec<u8>>,
content_type: Option<&str>,
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let mut request = http::Request::builder().method(method.clone()).uri(uri);
request = request.header(HOST, authority);
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if let Some(content_type) = content_type {
request = request.header(CONTENT_TYPE, content_type);
}
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
let mut request_builder = client.request(reqwest_method, url);
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if let Some(body) = body {
request_builder = request_builder.body(body);
}
Ok(request_builder.send().await?)
}
async fn signed_request_with_session_token(
method: http::Method,
url: &str,
access_key: &str,
secret_key: &str,
session_token: &str,
body: Option<Vec<u8>>,
content_type: Option<&str>,
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let mut request = http::Request::builder().method(method.clone()).uri(uri);
request = request.header(HOST, authority);
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if !session_token.is_empty() {
request = request.header("x-amz-security-token", session_token);
}
if let Some(content_type) = content_type {
request = request.header(CONTENT_TYPE, content_type);
}
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
let signed = sign_v4(
request.body(Body::empty())?,
content_len,
access_key,
secret_key,
session_token,
"us-east-1",
);
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
let client = local_http_client();
let mut request_builder = client.request(reqwest_method, url);
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if let Some(body) = body {
request_builder = request_builder.body(body);
}
Ok(request_builder.send().await?)
}
fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
let open = format!("<{tag}>");
let close = format!("</{tag}>");
@@ -1016,35 +904,6 @@ fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_k
Client::from_conf(config)
}
async fn admin_create_user(
env: &RustFSTestEnvironment,
username: &str,
secret_key: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
let body = serde_json::json!({
"secretKey": secret_key,
"status": "enabled"
});
let response = signed_request(
http::Method::PUT,
&url,
&env.access_key,
&env.secret_key,
Some(body.to_string().into_bytes()),
Some("application/json"),
)
.await?;
if response.status() != StatusCode::OK {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(format!("create user failed: {status} {body}").into());
}
Ok(())
}
async fn admin_add_canned_policy(
env: &RustFSTestEnvironment,
policy_name: &str,
crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs
+13 -24
View File
@@ -15,7 +15,6 @@
use crate::common::{RustFSTestClusterEnvironment, init_logging};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::CompletedMultipartUpload;
use tokio::time::{Duration, sleep};
use tracing::info;
use uuid::Uuid;
@@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload(
}
}
async fn complete_reports_missing_upload(
async fn multipart_listing_reports_missing_upload(
client: &aws_sdk_s3::Client,
bucket: &str,
key: &str,
upload_id: &str,
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
let result = client
.complete_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.multipart_upload(CompletedMultipartUpload::builder().build())
.send()
.await;
match result {
Ok(_) => Ok(false),
Err(SdkError::ServiceError(err)) => {
let code = err.err().meta().code().unwrap_or("");
if code == "NoSuchUpload" {
Ok(true)
} else {
Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
}
}
Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
}
let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
Ok(!result
.uploads()
.iter()
.any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
}
async fn wait_for_cleanup_on_all_nodes(
@@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes(
let mut all_cleaned = true;
for (idx, client) in clients.iter().enumerate() {
let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
if !(list_parts_missing && complete_missing) {
let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
if !(list_parts_missing && listing_missing) {
info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
all_cleaned = false;
break;
@@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
1,
"multipart upload should be visible before background cleanup"
);
assert!(
!multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
"multipart upload listing should contain the upload before background cleanup"
);
wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
crates/ecstore/AGENTS.md
-1
View File
@@ -49,4 +49,3 @@ Applies to `crates/ecstore/`.
## Suggested Validation
- `cargo test -p rustfs-ecstore`
- Full gate before commit: `make pre-commit`
crates/ecstore/src/api/mod.rs
+2 -2
View File
@@ -317,8 +317,6 @@ pub mod config {
}
pub mod data_usage {
#[cfg(feature = "test-util")]
pub use crate::data_usage::seed_bucket_usage_memory_for_test;
pub use crate::data_usage::{
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -330,6 +328,8 @@ pub mod data_usage {
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
store_data_usage_in_backend,
};
#[cfg(feature = "test-util")]
pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
}
pub mod disk {
crates/ecstore/src/bucket/bucket_target_sys.rs
+39 -1
View File
@@ -866,7 +866,7 @@ impl BucketTargetSys {
return Some(cli);
}
// TODO: spawn a task to reload the target
// TODO(backlog): spawn an async task to proactively reload the replication target
if self.is_reloading_target(bucket, arn).await {
return None;
}
@@ -3425,6 +3425,44 @@ mod tests {
assert!(mutexes.contains_key("second"));
}
#[tokio::test]
async fn update_all_targets_publishes_disable_proxy_on_target_client() {
// The read-proxy selector (replication_proxy::get_proxy_targets) skips
// targets whose TargetClient carries disable_proxy — the persisted
// per-target opt-out must survive client publication.
let sys = BucketTargetSys::default();
let target = |arn: &str, disable_proxy: bool| BucketTarget {
arn: arn.to_string(),
endpoint: "192.168.1.10:9000".to_string(),
target_bucket: "target-bucket".to_string(),
region: "us-east-1".to_string(),
disable_proxy,
credentials: Some(Credentials {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
session_token: None,
expiration: None,
}),
..Default::default()
};
let targets = BucketTargets {
targets: vec![target("arn:proxied", false), target("arn:opted-out", true)],
};
sys.update_all_targets("bucket", Some(&targets)).await;
let proxied = sys
.get_remote_target_client("bucket", "arn:proxied")
.await
.expect("client should be published");
assert!(!proxied.disable_proxy);
let opted_out = sys
.get_remote_target_client("bucket", "arn:opted-out")
.await
.expect("client should be published");
assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn target_updates_serialize_client_build_through_publication_per_bucket() {
let sys = Arc::new(BucketTargetSys::default());
crates/ecstore/src/bucket/replication/replication_pool.rs
+1 -1
View File
@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
.map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
let mut rstate = oi.replication_state();
rstate.replicate_decision_str = dsc.to_string();
let asz = oi.get_actual_size().unwrap_or_default();
let asz = oi.get_actual_size_or_physical();
let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
let checksum = if ssec { oi.checksum.clone() } else { None };
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+1 -1
View File
@@ -1412,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
};
let mut replication_state = oi.replication_state();
replication_state.replicate_decision_str = dsc.to_string();
let actual_size = oi.get_actual_size().unwrap_or_default();
let actual_size = oi.get_actual_size_or_physical();
Ok(ReplicateObjectInfo {
name: oi.name.clone(),
crates/ecstore/src/bucket/replication/replication_target_boundary.rs
+15 -1
View File
@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
.map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
version_id: object_info.version_id.map(|version_id| version_id.to_string()),
etag: object_info.etag.as_deref(),
actual_size: object_info.get_actual_size().unwrap_or_default(),
actual_size: object_info.get_actual_size_or_physical(),
delete_marker: object_info.delete_marker,
content_type: object_info.content_type.as_deref(),
content_encoding: object_info.content_encoding.as_deref(),
@@ -542,6 +542,20 @@ mod tests {
assert!(replication_target_head_is_newer_null_version(&source, &target));
}
#[test]
fn replication_source_uses_physical_size_for_unknown_compressed_object() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
let source = ObjectInfo {
size: 128,
actual_size: -1,
user_defined: Arc::new(metadata),
..Default::default()
};
assert_eq!(replication_source_object(&source).actual_size, 128);
}
#[test]
fn replication_target_head_content_matches_compare_etag_only() {
let source = ObjectInfo {
crates/ecstore/src/cluster/rpc/peer_s3_client.rs
+1 -1
View File
@@ -454,7 +454,7 @@ impl S3PeerSys {
}
}
topology_complete &= bucket_map.values().all(|count| *count >= quorum);
// TODO: MRF
// TODO(backlog): integrate MRF backlog stats into scanner bucket listing
}
let mut buckets: Vec<BucketInfo> = result_map.into_values().collect();
crates/ecstore/src/cluster/rpc/remote_disk.rs
+174 -24
View File
@@ -42,18 +42,19 @@ use futures::lock::Mutex;
use metrics::counter;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
};
use rustfs_protos::ChannelClass;
use rustfs_protos::evict_failed_connection;
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest,
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, node_service_client::NodeServiceClient,
};
@@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const BATCH_METADATA_RPC_OFF: &str = "off";
const BATCH_METADATA_RPC_AUTO: &str = "auto";
const BATCH_METADATA_RPC_ON: &str = "on";
@@ -112,6 +114,28 @@ const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
fn decode_delete_versions_errors(response: DeleteVersionsResponse, expected_len: usize) -> Vec<Option<Error>> {
if !response.item_errors.is_empty() {
if response.item_errors.len() != expected_len {
return vec![Some(Error::other("malformed delete_versions item errors")); expected_len];
}
return response
.item_errors
.into_iter()
.map(|error| (error.code != 0).then(|| error.into()))
.collect();
}
if response.errors.len() != expected_len {
return vec![Some(Error::other("malformed delete_versions errors")); expected_len];
}
response
.errors
.into_iter()
.map(|error| (!error.is_empty()).then(|| Error::other(error)))
.collect()
}
fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
if !response.success {
return Err(response.error.unwrap_or_default().into());
@@ -180,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
}
fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
.or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
.as_deref()
.map(parse_batch_metadata_rpc_mode)
@@ -1804,6 +1829,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
}
}
fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
if let Some(started_at) = started_at {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
}
}
/// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
/// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
/// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
@@ -1914,6 +1945,27 @@ fn decode_batch_read_version_response_items(
Ok(batch_read_version_resps)
}
fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
req.items
.iter()
.fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
total
.saturating_add(item.org_volume.len())
.saturating_add(item.volume.len())
.saturating_add(item.path.len())
.saturating_add(item.version_id.len())
})
}
fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
response
.batch_read_version_resps
.iter()
.map(String::len)
.sum::<usize>()
.saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
}
fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
file_info.validate_for_metadata_read().map_err(Into::into)
}
@@ -2406,8 +2458,6 @@ impl DiskAPI for RemoteDisk {
return errors;
}
// TODO: use Error not string
let result = self
.execute_with_timeout(
|| async {
@@ -2439,17 +2489,7 @@ impl DiskAPI for RemoteDisk {
}
return errors;
}
response
.errors
.iter()
.map(|error| {
if error.is_empty() {
None
} else {
Some(Error::other(error.to_string()))
}
})
.collect()
decode_delete_versions_errors(response, versions.len())
}
#[tracing::instrument(level = "trace", skip_all)]
@@ -2827,14 +2867,19 @@ impl DiskAPI for RemoteDisk {
state = "started",
"Remote disk RPC started"
);
let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let batch_read_version_req = compat_json(&req)?;
let batch_read_version_req_bin = encode_msgpack(&req)?;
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
let request_payload_bytes = batch_read_version_attribution_enabled
.then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
let batch_result = self
.execute_with_timeout_for_op(
"batch_read_version",
move || async move {
let disk = self.disk_ref().await;
let disk_len = disk.len();
let mut client = self
.get_bulk_client()
.await
@@ -2845,9 +2890,20 @@ impl DiskAPI for RemoteDisk {
batch_read_version_req_bin: batch_read_version_req_bin.into(),
});
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
if let Some(request_payload_bytes) = request_payload_bytes {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
request_payload_bytes.saturating_add(disk_len),
);
}
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let response = match client.batch_read_version(request).await {
Ok(response) => response.into_inner(),
Ok(response) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
response.into_inner()
}
Err(status) if status.code() == Code::Unimplemented => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
if mode.should_fallback_on_unimplemented() {
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
warn!(
@@ -2864,6 +2920,7 @@ impl DiskAPI for RemoteDisk {
}
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
@@ -2876,14 +2933,33 @@ impl DiskAPI for RemoteDisk {
);
return Err(Error::from(status));
}
Err(status) => return Err(Error::from(status)),
Err(status) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(Error::from(status));
}
};
if !response.success {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(response.error.unwrap_or_default().into());
}
decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
batch_read_version_response_payload_len(&response),
);
let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
match decode_batch_read_version_response_items(response, &self.endpoint) {
Ok(batch_read_version_resps) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
Ok(Some(batch_read_version_resps))
}
Err(err) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
Err(err)
}
}
},
get_max_timeout_duration(),
)
@@ -3760,6 +3836,63 @@ mod tests {
static INIT: Once = Once::new();
#[test]
fn delete_versions_response_preserves_typed_item_errors() {
let errors = decode_delete_versions_errors(
DeleteVersionsResponse {
success: true,
errors: vec!["file not found".to_string(), String::new()],
error: None,
item_errors: vec![
rustfs_protos::proto_gen::node_service::Error {
code: DiskError::FileNotFound.to_u32(),
error_info: "file not found".to_string(),
},
rustfs_protos::proto_gen::node_service::Error::default(),
],
},
2,
);
assert!(matches!(errors.as_slice(), [Some(DiskError::FileNotFound), None]));
}
#[test]
fn delete_versions_response_accepts_legacy_string_errors() {
let errors = decode_delete_versions_errors(
DeleteVersionsResponse {
success: true,
errors: vec!["legacy error".to_string(), String::new()],
error: None,
item_errors: Vec::new(),
},
2,
);
assert_eq!(errors.len(), 2);
assert_eq!(errors[0].as_ref().map(ToString::to_string).as_deref(), Some("io error legacy error"));
assert!(errors[1].is_none());
}
#[test]
fn delete_versions_response_rejects_misaligned_item_errors() {
let errors = decode_delete_versions_errors(
DeleteVersionsResponse {
success: true,
errors: vec!["file not found".to_string()],
error: None,
item_errors: vec![rustfs_protos::proto_gen::node_service::Error {
code: DiskError::FileNotFound.to_u32(),
error_info: "file not found".to_string(),
}],
},
2,
);
assert_eq!(errors.len(), 2);
assert!(errors.iter().all(Option::is_some));
}
#[test]
fn disk_mutation_digest_marks_rolling_compatibility() {
let mut request = Request::new(());
@@ -4554,6 +4687,7 @@ mod tests {
} else {
"file version not found".to_string()
},
error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
}
}
@@ -4673,6 +4807,7 @@ mod tests {
fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
@@ -4682,10 +4817,25 @@ mod tests {
);
}
#[test]
fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
],
|| {
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
},
);
}
#[test]
fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
crates/ecstore/src/cluster/rpc/runtime_sources.rs
+57 -3
View File
@@ -14,9 +14,10 @@
use rustfs_io_metrics::internode_metrics::{
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
global_internode_metrics,
};
use std::time::Duration;
@@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_stage_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
stage,
duration,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics()
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
}
pub(crate) fn record_remote_disk_grpc_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
crates/ecstore/src/config/storageclass.rs
+4 -1
View File
@@ -248,10 +248,13 @@ impl Config {
let shard_size = shard_size as usize;
// Keep the historical two-data-shard object budget while preventing
// wider EC layouts from multiplying the maximum inline object size.
// Use div_ceil to match the shard_file_size calculation (which also uses
// div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline
// for objects right at the threshold.
let inline_block = if self.initialized && self.inline_block_explicit {
self.inline_block
} else {
(DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK)
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
};
if versioned {
crates/ecstore/src/core/pools.rs
+2312 -564
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/core/sets.rs
+84 -69
View File
@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
range::HTTPRangeSpec,
};
use crate::{
@@ -249,7 +249,7 @@ impl Sets {
self.connect_disks().await;
// TODO: config interval
// TODO(backlog): make monitor_and_connect interval configurable instead of hardcoded 15s
let mut interval = tokio::time::interval(Duration::from_secs(15));
loop {
tokio::select! {
@@ -414,6 +414,66 @@ fn apply_delete_objects_results(
}
}
fn apply_delete_accounting_results(
accounting: &mut [Option<DeleteAccounting>],
set_objects: &[DelObj],
set_accounting: &[Option<DeleteAccounting>],
) {
for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
accounting[obj.orig_idx] = value.clone();
}
}
impl Sets {
pub(crate) async fn delete_objects_with_accounting(
&self,
bucket: &str,
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut del_errs = vec![None; objects.len()];
let mut accounting = vec![None; objects.len()];
let mut set_obj_map = HashMap::new();
for (i, obj) in objects.iter().enumerate() {
let idx = self.get_hashed_set_index(obj.object_name.as_str());
set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
orig_idx: i,
obj: obj.clone(),
});
}
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
let mut futures = FuturesUnordered::new();
let bucket = bucket.to_owned();
for (set_index, set_objects) in set_obj_map {
let disks = self.get_disks(set_index);
let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
let bucket = bucket.clone();
let opts = opts.clone();
let semaphore = semaphore.clone();
futures.push(async move {
let _permit = semaphore
.acquire_owned()
.await
.expect("delete_objects semaphore should remain open");
let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
(set_objects, deleted, errors, accounting)
});
}
while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
}
(del_objects, del_errs, accounting)
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::object::ObjectIO for Sets {
type Error = Error;
@@ -655,65 +715,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
// Default return value
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut del_errs = Vec::with_capacity(objects.len());
for _ in 0..objects.len() {
del_errs.push(None)
}
let mut set_obj_map = HashMap::new();
// hash key
for (i, obj) in objects.iter().enumerate() {
let idx = self.get_hashed_set_index(obj.object_name.as_str());
if !set_obj_map.contains_key(&idx) {
set_obj_map.insert(
idx,
vec![DelObj {
// set_idx: idx,
orig_idx: i,
obj: obj.clone(),
}],
);
} else if let Some(val) = set_obj_map.get_mut(&idx) {
val.push(DelObj {
// set_idx: idx,
orig_idx: i,
obj: obj.clone(),
});
}
}
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
let mut futures = FuturesUnordered::new();
let bucket = bucket.to_string();
for (k, v) in set_obj_map {
let disks = self.get_disks(k);
let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
let bucket = bucket.clone();
let opts = opts.clone();
let semaphore = semaphore.clone();
futures.push(async move {
let _permit = semaphore
.acquire_owned()
.await
.expect("delete_objects semaphore should remain open");
let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
(v, dobjects, errs)
});
}
while let Some((v, dobjects, errs)) = futures.next().await {
apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
}
(del_objects, del_errs)
let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
(deleted, errors)
}
#[tracing::instrument(skip(self))]
@@ -985,14 +988,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::heal::HealOperations for Sets {
type Error = Error;
type HealResultItem = HealResultItem;
type HealOptions = HealOpts;
#[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
impl Sets {
pub(crate) async fn heal_format_with_fence<F>(&self, dry_run: bool, fence_lost: F) -> Result<(HealResultItem, Option<Error>)>
where
F: Fn() -> bool + Send + Sync,
{
let (disks, init_errs) = init_storage_disks_with_errors(
&self.endpoints.endpoints,
&DiskOption {
@@ -1065,6 +1065,9 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
// Save new formats `format.json` on unformatted disks.
for (index, (fm, disk)) in tmp_new_formats.iter_mut().zip(disks.iter()).enumerate() {
if fm.is_some() && disk.is_some() {
if fence_lost() {
return Ok((res, Some(StorageError::SlowDown)));
}
if let Err(err) = save_format_file(disk, fm).await {
if let Some(disk) = disk.as_ref() {
let _ = disk.close().await;
@@ -1098,6 +1101,18 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
}
Ok((res, None))
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::heal::HealOperations for Sets {
type Error = Error;
type HealResultItem = HealResultItem;
type HealOptions = HealOpts;
#[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
self.heal_format_with_fence(dry_run, || false).await
}
#[tracing::instrument(skip(self))]
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
let mut result = HealResultItem {
crates/ecstore/src/data_movement/mod.rs
+161 -42
View File
@@ -26,7 +26,7 @@ use crate::storage_api_contracts::{
namespace::NamespaceLocking as _,
object::{HTTPPreconditions, ObjectOperations as _},
};
use crate::store::ECStore;
use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -856,7 +856,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
is_unversioned_data_movement_object(source)
&& is_unversioned_data_movement_object(target)
&& !target.delete_marker
&& source
.mod_time
.zip(target.mod_time)
@@ -1028,6 +1027,7 @@ pub(crate) enum SourceCleanupError {
pub(crate) struct SourceCleanupBucketFence<'a> {
pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
}
fn ensure_source_cleanup_versions_match(
@@ -1065,7 +1065,9 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
struct SourceCleanupDeleteBarrierState {
bucket: String,
object: String,
fence_pending: tokio::sync::Notify,
arrived: tokio::sync::Notify,
is_paused: AtomicBool,
release: tokio::sync::Notify,
}
@@ -1079,7 +1081,7 @@ pub(crate) struct SourceCleanupDeleteBarrier {
}
#[cfg(test)]
static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<SourceCleanupDeleteBarrierState>>>> =
std::sync::OnceLock::new();
#[cfg(test)]
@@ -1092,15 +1094,22 @@ impl SourceCleanupDeleteBarrier {
let state = Arc::new(SourceCleanupDeleteBarrierState {
bucket: bucket.to_string(),
object: object.to_string(),
fence_pending: tokio::sync::Notify::new(),
arrived: tokio::sync::Notify::new(),
is_paused: AtomicBool::new(false),
release: tokio::sync::Notify::new(),
});
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison");
assert!(slot.is_none(), "source cleanup delete barrier must be unique");
*slot = Some(Arc::clone(&state));
assert!(
!barriers
.iter()
.any(|barrier| barrier.bucket == bucket && barrier.object == object),
"source cleanup delete barrier must be unique per object"
);
barriers.push(Arc::clone(&state));
Self { state }
}
@@ -1110,35 +1119,58 @@ impl SourceCleanupDeleteBarrier {
.expect("source cleanup should reach the pre-delete barrier");
}
pub(crate) async fn wait_until_fence_pending(&self) {
tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified())
.await
.expect("source cleanup should attempt the fixed mutation fence");
}
pub(crate) fn is_paused(&self) -> bool {
self.state.is_paused.load(Ordering::Acquire)
}
pub(crate) fn release(&self) {
self.state.release.notify_one();
}
}
#[cfg(test)]
pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) {
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison")
.iter()
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
.cloned();
if let Some(barrier) = barrier {
barrier.fence_pending.notify_one();
}
}
#[cfg(test)]
impl Drop for SourceCleanupDeleteBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison");
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
*slot = None;
}
barriers.retain(|state| !Arc::ptr_eq(state, &self.state));
}
}
#[cfg(test)]
async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
let barrier = SOURCE_CLEANUP_DELETE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison")
.as_ref()
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
.iter()
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
.cloned();
if let Some(barrier) = barrier {
barrier.is_paused.store(true, Ordering::Release);
barrier.arrived.notify_one();
barrier.release.notified().await;
}
@@ -1154,11 +1186,20 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
op_label: &str,
) -> std::result::Result<ObjectInfo, SourceCleanupError> {
let cleanup_key = encode_dir_object(object);
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
let _guard = ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(Error::from)?;
let source_guard = if bucket_fence
.object_mutation_fence
.is_some_and(SourceCleanupMutationFence::source_lock_covered)
{
None
} else {
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(Error::from)?,
)
};
if bucket_fence
.lifecycle_guard
@@ -1168,6 +1209,14 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
"{op_label}: bucket incarnation fence was lost before source cleanup"
))));
}
if bucket_fence
.object_mutation_fence
.is_some_and(SourceCleanupMutationFence::is_lock_lost)
{
return Err(SourceCleanupError::Storage(Error::other(format!(
"{op_label}: object mutation fence was lost before source cleanup"
))));
}
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
@@ -1182,7 +1231,12 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id,
..Default::default()
};
opts.add_namespace_lock_guard(&_guard);
if let Some(source_guard) = source_guard.as_ref() {
opts.add_namespace_lock_guard(source_guard);
}
if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
object_mutation_fence.add_namespace_lock_fence(&mut opts);
}
if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
}
@@ -1330,6 +1384,37 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str {
}
}
pub(crate) async fn migrate_decommission_object(
store: Arc<ECStore>,
pool_idx: usize,
bucket: String,
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
) -> Result<()> {
let source = rd.object_info.clone();
let _mutation_fence = store
.acquire_decommission_object_mutation_fence(&bucket, &source.name)
.await?;
let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
.await?
.ok_or(Error::FileNotFound)?;
if !is_equivalent_data_movement_object_identity(&source, &current, true, false) {
return Err(Error::FileNotFound);
}
migrate_object_inner(
store,
pool_idx,
bucket,
rd,
source_bucket_incarnation_id,
op_label,
Some(&_mutation_fence),
)
.await
}
pub(crate) async fn migrate_object(
store: Arc<ECStore>,
pool_idx: usize,
@@ -1337,6 +1422,18 @@ pub(crate) async fn migrate_object(
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
) -> Result<()> {
migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
}
async fn migrate_object_inner(
store: Arc<ECStore>,
pool_idx: usize,
bucket: String,
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
mutation_fence: Option<&ObjectLockDiagGuard>,
) -> Result<()> {
let object_info = rd.object_info.clone();
let has_part_checksums = object_info
@@ -1350,7 +1447,7 @@ pub(crate) async fn migrate_object(
let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts)
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
.await
{
Ok(res) => res,
@@ -1448,7 +1545,7 @@ pub(crate) async fn migrate_object(
if let Err(err) = store
.clone()
.complete_multipart_upload_for_data_movement(
target_pool_idx,
(target_pool_idx, mutation_fence),
&bucket,
&object_info.name,
&res.upload_id,
@@ -1609,7 +1706,7 @@ pub(crate) async fn migrate_object(
let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
let (target_pool_idx, put_result) = store
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts)
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
.await
.map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
if let Err(err) = put_result {
@@ -3541,25 +3638,47 @@ mod tests {
}
#[test]
fn test_precondition_conflict_rejects_newer_delete_marker() {
let source = ObjectInfo {
size: 128,
etag: Some("etag-source".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let target = ObjectInfo {
delete_marker: true,
etag: None,
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
..source.clone()
};
fn test_precondition_conflict_accepts_only_newer_null_delete_marker() {
for version_id in [None, Some(Uuid::nil())] {
let source = ObjectInfo {
version_id,
size: 128,
etag: Some("etag-source".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let target = ObjectInfo {
delete_marker: true,
etag: None,
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
..source.clone()
};
let should_resume =
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
.expect("delete marker conflict should be evaluated");
assert!(
resolve_data_movement_overwrite_resume_result(
&Error::PreconditionFailed,
Ok(Some(target.clone())),
&source,
0,
1,
)
.expect("newer null delete marker should be evaluated")
);
assert!(!should_resume);
let mut same_time = target.clone();
same_time.mod_time = source.mod_time;
assert!(
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,)
.expect("same-generation null delete marker should be rejected")
);
let mut versioned = target;
versioned.version_id = Some(Uuid::new_v4());
assert!(
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,)
.expect("a UUID delete marker must not erase a null source version")
);
}
}
#[test]
crates/ecstore/src/data_usage/mod.rs
+108 -8
View File
@@ -1391,7 +1391,37 @@ impl BucketUsageAccumulator {
}
pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
// A compressed object may carry -1 while the transformed size is unknown
// (legacy streaming sentinel). In that case the persisted physical size
// is still a valid accounting floor; every other negative value is corrupt.
// An explicit negative `actual-size` metadata value is corrupt, however:
// the sentinel is only valid in the in-memory/object-part field written by
// the legacy streaming path, not as a persisted declared size.
let compressed = object.is_compressed();
if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
return Err(Error::PartMissingOrCorrupt);
}
if object
.parts
.iter()
.any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
{
return Err(Error::PartMissingOrCorrupt);
}
let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
.filter(|value| !value.is_empty());
if declared_actual_size
.as_deref()
.and_then(|value| value.parse::<i64>().ok())
.is_some_and(|size| size < 0)
{
return Err(Error::PartMissingOrCorrupt);
}
let logical_size = match object.get_actual_size().map_err(Error::other)? {
size if size == -1 && compressed && declared_actual_size.is_none() => None,
size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
_ => return Err(Error::PartMissingOrCorrupt),
};
let persisted_part_size = if object.parts.is_empty() {
u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
} else {
@@ -1399,12 +1429,8 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
// Compressed streaming objects persist -1 when the transformed
// part size is unknown. The physical part size remains a valid
// quota floor; reject only non-negative values that overflow.
let actual_size = if part.actual_size < 0 {
if object.is_compressed() {
0
} else {
return Err(Error::PartMissingOrCorrupt);
}
let actual_size = if part.actual_size == -1 {
0
} else {
u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
};
@@ -1412,7 +1438,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
})?
};
Ok(logical_size.max(persisted_part_size))
Ok(logical_size.unwrap_or(0).max(persisted_part_size))
}
type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -3320,6 +3346,80 @@ mod tests {
);
}
#[test]
fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
let object = ObjectInfo {
size: 400,
actual_size: -1,
user_defined: Arc::new(metadata),
..Default::default()
};
assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
}
#[test]
fn quota_object_size_rejects_compressed_part_sum_overflow() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
let object = ObjectInfo {
size: 1,
user_defined: Arc::new(metadata),
parts: Arc::new(vec![
rustfs_filemeta::ObjectPartInfo {
actual_size: i64::MAX,
..Default::default()
},
rustfs_filemeta::ObjectPartInfo {
actual_size: 1,
..Default::default()
},
]),
..Default::default()
};
assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
}
#[test]
fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
let corrupt_object = ObjectInfo {
size: 400,
actual_size: -2,
user_defined: Arc::new(metadata.clone()),
..Default::default()
};
assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
let corrupt_part = ObjectInfo {
size: 400,
user_defined: Arc::new(metadata),
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
size: 400,
actual_size: -2,
..Default::default()
}]),
..Default::default()
};
assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
}
#[tokio::test]
#[serial]
async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
crates/ecstore/src/disk/local.rs
+62 -31
View File
@@ -858,6 +858,7 @@ const EVENT_DISK_LOCAL_DIRECT_IO_FALLBACK: &str = "disk_local_direct_io_fallback
#[cfg(target_os = "linux")]
const EVENT_DISK_LOCAL_URING_LATCH_OFF: &str = "disk_local_uring_latch_off";
const EVENT_DISK_LOCAL_DELETE_FAILED: &str = "disk_local_delete_failed";
const EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED: &str = "disk_local_delete_rollback_failed";
const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
@@ -5215,8 +5216,8 @@ impl LocalDisk {
let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default());
// TODO: DIRECT support
// TODD: DiskInfo
// TODO(backlog): add O_DIRECT I/O support for performance-critical paths
// TODO(backlog): populate DiskInfo in constructor
let mut disk = Self {
root: root.clone(),
publication_root,
@@ -5751,7 +5752,7 @@ impl LocalDisk {
// return Ok(());
// TODO: async notifications for disk space checks and trash cleanup
// TODO(backlog): make disk space checks and trash cleanup event-driven instead of poll-based
let trash_path = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
// if let Some(parent) = trash_path.parent() {
@@ -5997,7 +5998,7 @@ impl LocalDisk {
#[hotpath::measure(impl_type = "LocalDisk")]
async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
// TODO: timeout support
// TODO(backlog): add configurable timeout for read_all_data operations
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
Ok(data)
}
@@ -6106,6 +6107,43 @@ impl LocalDisk {
Ok((bytes, modtime))
}
async fn write_missing_delete_marker(
&self,
volume: &str,
path: &str,
fi: FileInfo,
object_dir: &Path,
xl_path: &Path,
rollback_dir: Option<Uuid>,
) -> Result<()> {
if let Some(rollback_dir) = rollback_dir {
let rollback_path = object_dir.join(rollback_dir.to_string());
fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
.await
.map_err(to_file_error)?;
}
if let Err(err) = self.write_metadata("", volume, path, fi).await {
if let Some(rollback_dir) = rollback_dir
&& let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await
{
warn!(
event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
result = "failed",
volume,
path,
rollback_dir = %rollback_dir,
error = ?restore_err,
"Disk local delete rollback failed"
);
}
return Err(err);
}
Ok(())
}
async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
let volume_dir = self.io_get_bucket_path(volume)?;
let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
@@ -6123,7 +6161,20 @@ impl LocalDisk {
return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
}
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?;
let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
Ok(data) => data,
Err(DiskError::FileNotFound) => {
// `deleted` alone can be an explicit marker purge; only
// `mark_deleted` may create metadata that was not present.
let Some(delete_marker) = fis.iter().find(|fi| fi.deleted && fi.mark_deleted).cloned() else {
return Err(DiskError::FileNotFound);
};
return self
.write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir)
.await;
}
Err(err) => return Err(err),
};
if data.is_empty() {
return Err(DiskError::FileNotFound);
@@ -6674,7 +6725,7 @@ impl LocalDisk {
return Ok(());
}
// TODO: add lock
// TODO(backlog): add directory listing lock to prevent concurrent enumeration
let stall = opts.stall_timeout_duration();
@@ -8796,7 +8847,7 @@ impl DiskAPI for LocalDisk {
Ok(entries)
}
// FIXME: TODO: io.writer TODO cancel
// TODO(backlog): support io.writer cancellation and early termination in walk_dir
#[tracing::instrument(level = "trace", skip_all)]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
self.wait_for_startup_cleanup().await;
@@ -9880,7 +9931,7 @@ impl DiskAPI for LocalDisk {
);
return Err(e);
}
// TODO: health check
// TODO(backlog): add post-setup disk health verification
}
Ok(())
}
@@ -10422,29 +10473,9 @@ impl DiskAPI for LocalDisk {
}
if fi.deleted && force_del_marker {
if let Some(rollback_dir) = rollback_dir {
let rollback_path = file_path.join(rollback_dir.to_string());
fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
.await
.map_err(to_file_error)?;
}
if let Err(err) = self.write_metadata("", volume, path, fi).await {
if let Some(rollback_dir) = rollback_dir
&& let Err(restore_err) =
restore_delete_rollback(file_path.as_path(), &xl_path, rollback_dir, &self.publication_root).await
{
warn!(
volume,
path,
rollback_dir = %rollback_dir,
error = ?restore_err,
"failed to restore metadata after delete marker commit error"
);
}
return Err(err);
}
return Ok(());
return self
.write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir)
.await;
}
return if fi.version_id.is_some() {
crates/ecstore/src/disk/mod.rs
+69 -25
View File
@@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_DISK: &str = "disk";
const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;
pub fn part_transaction_path(part_path: &str) -> String {
match part_path.rsplit_once('/') {
@@ -62,6 +64,7 @@ use bytes::Bytes;
use endpoint::Endpoint;
use error::DiskError;
use error::{Error, Result};
use futures::stream::{self, StreamExt};
use local::LocalDisk;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_madmin::info_commands::DiskMetrics;
@@ -417,6 +420,14 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
match self {
Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
match self {
@@ -1028,36 +1039,47 @@ where
D: DiskAPI + ?Sized,
{
validate_batch_read_version_item_count(req.items.len())?;
let parallelism = batch_read_version_server_parallelism();
let mut responses = Vec::with_capacity(req.items.len());
for (index, item) in req.items.iter().enumerate() {
let response = match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: true,
file_info,
error: String::new(),
},
Err(err) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
},
};
responses.push(response);
}
let mut responses = stream::iter(req.items.into_iter().enumerate())
.map(|(index, item)| async move {
match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: true,
file_info,
error: String::new(),
error_code: 0,
},
Err(err) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
error_code: err.to_u32(),
},
}
})
.buffer_unordered(parallelism)
.collect::<Vec<_>>()
.await;
responses.sort_unstable_by_key(|response| response.index);
Ok(responses)
}
fn batch_read_version_server_parallelism() -> usize {
rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
.clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CheckPartsResp {
pub results: Vec<usize>,
@@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp {
pub success: bool,
pub file_info: FileInfo,
pub error: String,
#[serde(default)]
pub error_code: u32,
}
pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
@@ -1417,6 +1441,26 @@ mod tests {
assert!(!partial_valid_location.valid());
}
#[test]
fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
assert_eq!(batch_read_version_server_parallelism(), 4);
});
}
#[test]
fn batch_read_version_server_parallelism_honors_env_with_bounds() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
assert_eq!(batch_read_version_server_parallelism(), 8);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
assert_eq!(batch_read_version_server_parallelism(), 1);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
});
}
/// Test FileInfoVersions find_version_index
#[test]
fn test_file_info_versions_find_version_index() {
crates/ecstore/src/disk/os.rs
+42 -4
View File
@@ -315,7 +315,7 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
#[cfg(unix)]
{
let dir = dir.as_ref().to_path_buf();
tokio::task::spawn_blocking(move || fsync_dir_std(dir)).await?
fsync_spawn_blocking(move || fsync_dir_std(dir)).await?
}
#[cfg(not(unix))]
@@ -683,7 +683,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
#[cfg(test)]
let dir = group.dir.clone();
let dir_file = group.dir_file.clone();
tokio::task::spawn_blocking(move || {
fsync_spawn_blocking(move || {
#[cfg(test)]
{
if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -1080,6 +1080,44 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
/// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
/// configured with >1 threads, isolates device-bound fsync from the main
/// blocking pool so reads (pread/stat/open) are not starved. `None` means
/// fall back to the main runtime (zero behavior change).
static FSYNC_RUNTIME: LazyLock<Option<tokio::runtime::Runtime>> = LazyLock::new(|| {
let threads =
rustfs_utils::get_env_usize(rustfs_config::ENV_FSYNC_BLOCKING_THREADS, rustfs_config::DEFAULT_FSYNC_BLOCKING_THREADS);
if threads <= 1 {
return None;
}
let mut builder = tokio::runtime::Builder::new_multi_thread();
builder
.worker_threads(num_cpus::get().min(8))
.max_blocking_threads(threads)
.thread_name("rustfs-fsync")
.thread_stack_size(512 * 1024)
.enable_all();
match builder.build() {
Ok(rt) => {
tracing::info!(threads, "fsync dedicated blocking pool enabled");
Some(rt)
}
Err(err) => {
tracing::warn!(%err, "failed to build fsync runtime, falling back to main pool");
None
}
}
});
/// Spawn a blocking task on the fsync-dedicated runtime if configured,
/// otherwise fall back to the main tokio blocking pool.
fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle<T> {
match FSYNC_RUNTIME.as_ref() {
Some(rt) => rt.spawn_blocking(f),
None => tokio::task::spawn_blocking(f),
}
}
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
type NamespaceMutationLock = AsyncMutex<()>;
@@ -1217,7 +1255,7 @@ where
F: FnOnce() -> io::Result<T> + Send + 'static,
{
let (disk_permit, global_permit) = acquire_file_sync_permits(disk_permits).await?;
let result = tokio::task::spawn_blocking(move || {
let result = fsync_spawn_blocking(move || {
let _disk_permit = disk_permit;
work()
})
@@ -2146,7 +2184,7 @@ async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'stati
wait_started,
);
let disk_permit = admission.disk_permit.clone();
let result = tokio::task::spawn_blocking(move || {
let result = fsync_spawn_blocking(move || {
let _lease = lease;
let _disk_permit = disk_permit;
operation()
crates/ecstore/src/layout/endpoints.rs
+2 -2
View File
@@ -249,7 +249,7 @@ impl PoolEndpointList {
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
// TODO Check for cross device mounts if any.
// TODO(backlog): check for cross-device mounts in single-drive setup
return Ok(Self {
inner: vec![Endpoints::from(vec![endpoint])],
@@ -264,7 +264,7 @@ impl PoolEndpointList {
// Convert args to endpoints
let mut eps = Endpoints::try_from(set_layout.as_slice())?;
// TODO Check for cross device mounts if any.
// TODO(backlog): check for cross-device mounts in multi-pool setup
for (disk_idx, ep) in eps.as_mut().iter_mut().enumerate() {
ep.set_pool_index(pool_idx);
crates/ecstore/src/lib.rs
+8
View File
@@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() {
cluster::rpc::shutdown_background_monitors();
}
/// Publish that the process is ready to serve user-object GET traffic.
///
/// Experimental metadata coalescing is allowed to run only after this point so
/// startup and internal metadata reads keep the original per-disk path.
pub fn mark_get_metadata_read_version_coalescing_service_ready() {
runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
}
#[cfg(test)]
mod rio_tests {
#[test]
crates/ecstore/src/object_api/types.rs
+55 -15
View File
@@ -24,7 +24,7 @@ use crate::storage_api_contracts::{
pub struct NamespaceLockFence {
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
#[cfg(test)]
forced_lost: Arc<std::sync::atomic::AtomicBool>,
forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
}
impl Debug for NamespaceLockFence {
@@ -40,13 +40,17 @@ impl NamespaceLockFence {
Self {
signals: Arc::default(),
#[cfg(test)]
forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)),
forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
}
}
pub(crate) fn is_lock_lost(&self) -> bool {
#[cfg(test)]
if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
if self
.forced_lost
.iter()
.any(|lost| lost.load(std::sync::atomic::Ordering::Acquire))
{
return true;
}
self.signals.iter().any(|signal| signal.is_lost())
@@ -57,27 +61,26 @@ impl NamespaceLockFence {
}
fn extend(&mut self, other: &Self) {
if Arc::ptr_eq(&self.signals, &other.signals) {
return;
if !Arc::ptr_eq(&self.signals, &other.signals) {
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
}
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
#[cfg(test)]
if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
self.forced_lost.store(true, std::sync::atomic::Ordering::Release);
if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
}
}
#[cfg(test)]
pub(crate) fn lost_for_test() -> Self {
let fence = Self::new();
fence.forced_lost.store(true, std::sync::atomic::Ordering::Release);
fence.forced_lost[0].store(true, std::sync::atomic::Ordering::Release);
fence
}
#[cfg(test)]
pub(crate) fn loss_handle_for_test() -> (Self, Arc<std::sync::atomic::AtomicBool>) {
let fence = Self::new();
(fence.clone(), Arc::clone(&fence.forced_lost))
(fence.clone(), Arc::clone(&fence.forced_lost[0]))
}
}
@@ -411,6 +414,13 @@ impl ObjectOptions {
self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
}
#[cfg(test)]
pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
self.namespace_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
.extend(fence);
}
pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
self.lifecycle_delete_all_journal
.get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
@@ -689,6 +699,9 @@ impl ObjectInfo {
}
pub fn get_actual_size(&self) -> std::io::Result<i64> {
if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
return Err(std::io::Error::other("invalid negative actual size"));
}
if self.actual_size > 0 {
return Ok(self.actual_size);
}
@@ -700,10 +713,25 @@ impl ObjectInfo {
let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
return Ok(size);
}
let mut actual_size = 0;
self.parts.iter().for_each(|part| {
actual_size += part.actual_size;
});
if self.actual_size == -1 && self.parts.is_empty() {
return Ok(-1);
}
let mut actual_size = 0_i64;
let mut unknown = false;
for part in self.parts.iter() {
match part.actual_size {
-1 => unknown = true,
size if size >= 0 => {
actual_size = actual_size
.checked_add(size)
.ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
}
_ => return Err(std::io::Error::other("invalid negative compressed part size")),
}
}
if unknown {
return Ok(-1);
}
if actual_size == 0 && actual_size != self.size {
return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
}
@@ -718,6 +746,18 @@ impl ObjectInfo {
Ok(self.size)
}
/// Returns a non-negative size for client and replication boundaries.
///
/// Compressed legacy metadata can retain the internal `-1` unknown-size
/// sentinel. Those boundaries cannot emit a negative length, so they use
/// the persisted physical size while quota accounting keeps the sentinel
/// distinction in [`crate::data_usage::quota_object_size`].
pub fn get_actual_size_or_physical(&self) -> i64 {
self.get_actual_size()
.map(|size| if size >= 0 { size } else { self.size.max(0) })
.unwrap_or_else(|_| self.size.max(0))
}
pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
let mut version_id = fi.version_id;
@@ -1091,7 +1131,7 @@ impl ObjectInfo {
}
};
// TODO:VersionPurgeStatus
// TODO(backlog): handle VersionPurgeStatus in object listing
let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned));
crates/ecstore/src/runtime/global.rs
+14 -1
View File
@@ -25,7 +25,10 @@ use lazy_static::lazy_static;
use rustfs_lock::client::LockClient;
use std::{
collections::HashMap,
sync::{Arc, OnceLock},
sync::{
Arc, OnceLock,
atomic::{AtomicBool, Ordering},
},
time::SystemTime,
};
use tokio::sync::{OnceCell, RwLock};
@@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000;
pub const DISK_FILL_FRACTION: f64 = 0.99;
pub const DISK_RESERVE_FRACTION: f64 = 0.15;
static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
}
pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
}
// Global singletons for backward compatibility with MinIO port.
// These should be migrated to AppContext over time.
// See issue #730 for migration plan.
crates/ecstore/src/runtime/instance.rs
+9
View File
@@ -160,6 +160,10 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Serializes decommission data-movement operations with cancellation and
/// a subsequent restart. Readers are held across one object side effect;
/// the transition path takes the writer after cancelling the routine.
decommission_operation_gate: Arc<RwLock<()>>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -200,6 +204,7 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
decommission_operation_gate: Arc::new(RwLock::new(())),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -218,6 +223,10 @@ impl InstanceContext {
self.lock_manager.clone()
}
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.decommission_operation_gate)
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
crates/ecstore/src/services/metrics_realtime.rs
+12
View File
@@ -256,6 +256,10 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
cycle_max_objects: metrics.cycle_max_objects,
cycle_max_directories: metrics.cycle_max_directories,
cycle_timeout_total: metrics.cycle_timeout_total,
cycle_recovery_required_total: metrics.cycle_recovery_required_total,
cycle_last_progress_age: metrics.cycle_last_progress_age,
leader_lease_without_progress: metrics.leader_lease_without_progress,
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
@@ -611,6 +615,10 @@ mod test {
current_started: chrono_to_jiff_timestamp(current_started),
last_cycle_partial_source: "usage".to_string(),
last_cycle_partial_source_code: 1,
cycle_timeout_total: 3,
cycle_recovery_required_total: 2,
cycle_last_progress_age: 17,
leader_lease_without_progress: true,
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
source: "usage".to_string(),
cycles: 2,
@@ -622,6 +630,10 @@ mod test {
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
assert_eq!(scanner.last_cycle_partial_source, "usage");
assert_eq!(scanner.last_cycle_partial_source_code, 1);
assert_eq!(scanner.cycle_timeout_total, 3);
assert_eq!(scanner.cycle_recovery_required_total, 2);
assert_eq!(scanner.cycle_last_progress_age, 17);
assert!(scanner.leader_lease_without_progress);
let usage = scanner
.partial_cycles_by_source
.iter()
crates/ecstore/src/services/rebalance/entry.rs
+1
View File
@@ -334,6 +334,7 @@ impl ECStore {
lifecycle_guard: bucket_incarnation_fence
.as_ref()
.and_then(|guard| guard.namespace_lock_guard()),
..Default::default()
},
"rebalance",
),
crates/ecstore/src/set_disk/core/io_primitives.rs
+565 -7
View File
@@ -53,11 +53,12 @@ use crate::diagnostics::get::{
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::disk::disk_store::DiskStoreRenameDataExt;
use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout};
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
use crate::disk::{
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
};
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::ShardReader;
@@ -75,7 +76,7 @@ use std::{
future::Future,
pin::Pin,
sync::{
OnceLock,
Arc, OnceLock,
atomic::{AtomicUsize, Ordering},
},
task::{Context, Poll},
@@ -94,6 +95,242 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String {
[bucket, object].join("/")
}
fn read_version_coalescing_enabled() -> bool {
let enabled = || {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on"))
};
#[cfg(test)]
{
enabled()
}
#[cfg(not(test))]
{
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(enabled)
}
}
fn read_version_coalescing_delay() -> Duration {
#[cfg(test)]
{
let micros = rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
);
Duration::from_micros(micros)
}
#[cfg(not(test))]
{
static DELAY: OnceLock<Duration> = OnceLock::new();
*DELAY.get_or_init(|| {
Duration::from_micros(rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
))
})
}
}
struct CoalescedReadVersionRequest {
item: BatchReadVersionItem,
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
struct ReadVersionCoalescerKey {
disk: usize,
incl_free_versions: bool,
read_data: bool,
healing: bool,
}
impl ReadVersionCoalescerKey {
fn new(disk: &DiskStore, opts: &ReadOptions) -> Self {
Self {
disk: Arc::as_ptr(disk) as usize,
incl_free_versions: opts.incl_free_versions,
read_data: opts.read_data,
healing: opts.healing,
}
}
}
#[derive(Default)]
struct ReadVersionCoalescer {
lanes: HashMap<ReadVersionCoalescerKey, Vec<CoalescedReadVersionRequest>>,
}
fn read_version_coalescer() -> &'static Mutex<ReadVersionCoalescer> {
static COALESCER: OnceLock<Mutex<ReadVersionCoalescer>> = OnceLock::new();
COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default()))
}
fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
counter!(
METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL,
"event" => event,
"item_count" => item_count.to_string()
)
.increment(1);
}
async fn read_version_via_coalescer(
disk: DiskStore,
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
opts: &ReadOptions,
allow_coalescing: bool,
) -> disk::error::Result<FileInfo> {
if !allow_coalescing || !read_version_coalescing_enabled() {
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
if !matches!(disk.as_ref(), Disk::Remote(_)) {
record_read_version_coalescer_event("bypass_non_remote", 1);
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
let (tx, rx) = oneshot::channel();
let item = BatchReadVersionItem {
org_volume: org_bucket.to_string(),
volume: bucket.to_string(),
path: object.to_string(),
version_id: version_id.to_string(),
};
let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
let lane = coalescer.lanes.entry(lane_key).or_default();
let schedule_delayed_flush = lane.is_empty();
lane.push(CoalescedReadVersionRequest { item, tx });
if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
coalescer.lanes.remove(&lane_key)
} else if schedule_delayed_flush {
let disk = disk.clone();
let task_opts = *opts;
tokio::spawn(async move {
tokio::time::sleep(read_version_coalescing_delay()).await;
flush_read_version_coalescer_lane(lane_key, disk, task_opts).await;
});
None
} else {
None
}
};
if let Some(pending) = pending {
flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
}
rx.await
.unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
}
async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
coalescer.lanes.remove(&lane_key).unwrap_or_default()
};
flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await;
}
async fn flush_read_version_coalescer_pending(
lane_key: ReadVersionCoalescerKey,
disk: DiskStore,
opts: ReadOptions,
pending: Vec<CoalescedReadVersionRequest>,
) {
if pending.is_empty() {
return;
}
#[cfg(test)]
{
let mut observed_paths = HashSet::new();
for request in &pending {
if observed_paths.insert(request.item.path.as_str()) {
disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk);
}
}
}
let mut senders = Vec::with_capacity(pending.len());
let mut items = Vec::with_capacity(pending.len());
for request in pending {
senders.push(request.tx);
items.push(request.item);
}
let expected_items = items.clone();
record_read_version_coalescer_event("attempted_batch", items.len());
let result =
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
.await
{
Ok(result) => result,
Err(_) => Err(DiskError::Timeout),
};
match result {
Ok(responses) => {
let results = map_batch_read_version_responses(&expected_items, responses);
for (tx, result) in senders.into_iter().zip(results) {
let _ = tx.send(result);
}
}
Err(err) => {
let message = err.to_string();
for tx in senders {
let _ = tx.send(Err(DiskError::other(message.clone())));
}
}
}
}
fn map_batch_read_version_responses(
expected_items: &[BatchReadVersionItem],
responses: Vec<BatchReadVersionResp>,
) -> Vec<crate::disk::error::Result<FileInfo>> {
let mut results = (0..expected_items.len())
.map(|_| Err(DiskError::other("coalesced read_version response missing")))
.collect::<Vec<_>>();
let mut seen = vec![false; expected_items.len()];
for response in responses {
let Some(expected) = expected_items.get(response.index) else {
continue;
};
let Some(slot) = results.get_mut(response.index) else {
continue;
};
if seen[response.index] {
*slot = Err(DiskError::other("coalesced read_version response duplicate index"));
continue;
}
seen[response.index] = true;
if response.path != expected.path || response.version_id != expected.version_id {
*slot = Err(DiskError::other("coalesced read_version response identity mismatch"));
} else {
*slot = if response.success {
Ok(response.file_info)
} else {
Err(batch_read_version_response_error(response.error_code, response.error))
};
}
}
results
}
fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError {
match DiskError::from_u32(error_code) {
Some(DiskError::Io(_)) | None => DiskError::other(error),
Some(error) => error,
}
}
pub(in crate::set_disk) fn bounded_metadata_fanout_order(
bucket: &str,
object: &str,
@@ -133,11 +370,15 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
order
}
use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::RwLock;
use tokio::sync::{Mutex, RwLock, oneshot};
use tokio::task::JoinSet;
pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
/// Default reader-setup strategy for the GET read path (rustfs/backlog#1215,
/// #1159, #923).
@@ -2356,6 +2597,7 @@ impl SetDisks {
false,
true,
0,
false,
)
.await?;
Ok((ress, errors))
@@ -2386,6 +2628,36 @@ impl SetDisks {
true,
caller_allows_early_stop,
default_parity_count,
false,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
incl_free_versions: bool,
caller_allows_early_stop: bool,
default_parity_count: usize,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
Self::read_all_fileinfo_inner(
disks,
org_bucket,
bucket,
object,
version_id,
read_data,
false,
incl_free_versions,
true,
caller_allows_early_stop,
default_parity_count,
true,
)
.await
}
@@ -2408,6 +2680,7 @@ impl SetDisks {
// subset would fail write quorum (backlog#872 regression).
caller_allows_early_stop: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let early_stop_enabled =
caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled());
@@ -2424,6 +2697,7 @@ impl SetDisks {
healing,
incl_free_versions,
default_parity_count,
allow_coalescing,
)
.await;
}
@@ -2446,6 +2720,7 @@ impl SetDisks {
healing,
incl_free_versions,
observe,
allow_coalescing,
)
.await
}
@@ -2461,6 +2736,7 @@ impl SetDisks {
healing: bool,
incl_free_versions: bool,
observe: bool,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = observe.then(Instant::now);
let mut ress = Vec::with_capacity(disks.len());
@@ -2492,7 +2768,7 @@ impl SetDisks {
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
@@ -2559,6 +2835,7 @@ impl SetDisks {
healing: bool,
incl_free_versions: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = Instant::now();
let mut ress = vec![FileInfo::default(); disks.len()];
@@ -2607,7 +2884,7 @@ impl SetDisks {
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
@@ -5737,6 +6014,7 @@ pub(crate) mod disk_call_counters {
/// Kind label for the per-disk `read_version` metadata RPC.
pub const KIND_READ_VERSION: &str = "read_version";
pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version";
/// Registry key: (object, kind, disk_index).
type CountKey = (String, String, usize);
@@ -6460,6 +6738,286 @@ mod tests {
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn metadata_read_version_coalescer_bypasses_local_disks() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-local-bypass-bucket";
let object_a = "coalesced-local-object-a";
let object_b = "coalesced-local-object-b";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await;
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")),
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")),
],
async {
let calls = disk_call_counters::observe(object_a);
let disks_a = disks.clone();
let disks_b = disks.clone();
let read_a = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_a, "", bucket, object_a, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
tokio::task::yield_now().await;
let read_b = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_b, "", bucket, object_b, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
let (metadata_a, errs_a) = read_a
.await
.expect("first read task should not panic")
.expect("first coalesced read should resolve");
let (metadata_b, errs_b) = read_b
.await
.expect("second read task should not panic")
.expect("second coalesced read should resolve");
assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS);
assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS);
assert!(errs_a.iter().all(Option::is_none));
assert!(errs_b.iter().all(Option::is_none));
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"local disks still execute the ordinary per-disk read_version path"
);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"GET coalescing targets internode RPC count only and must not batch local disk reads"
);
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn metadata_read_version_coalescer_requires_get_object_intent() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-default-bypass-bucket";
let object = "default-bypass-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async {
let calls = disk_call_counters::observe(object);
let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false)
.await
.expect("default metadata read should resolve");
assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"non-GET metadata paths must bypass coalescer even when the env gate is enabled"
);
})
.await;
drop(dirs);
}
#[test]
fn batch_read_version_response_mapping_preserves_index_and_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-c".to_string(),
version_id: "v-c".to_string(),
},
];
let ok_file_info = FileInfo {
name: "object-a".to_string(),
..Default::default()
};
let responses = vec![
BatchReadVersionResp {
index: 2,
path: "object-c".to_string(),
version_id: "v-c".to_string(),
success: false,
file_info: FileInfo::default(),
error: "disk read failed".to_string(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: ok_file_info,
error: String::new(),
error_code: 0,
},
];
let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
let first = results
.next()
.expect("slot 0 should exist")
.expect("slot 0 should map the success response by index");
assert_eq!(first.name, "object-a");
let missing = results
.next()
.expect("slot 1 should exist")
.expect_err("slot 1 should stay missing");
assert!(
missing.to_string().contains("response missing"),
"unexpected missing response error: {missing}"
);
let failed = results
.next()
.expect("slot 2 should exist")
.expect_err("slot 2 should map the response error");
assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}");
assert!(results.next().is_none());
}
#[test]
fn batch_read_version_response_mapping_preserves_typed_not_found_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
];
let results = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileNotFound.to_string(),
error_code: DiskError::FileNotFound.to_u32(),
},
BatchReadVersionResp {
index: 1,
path: "object-b".to_string(),
version_id: "v-b".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileVersionNotFound.to_string(),
error_code: DiskError::FileVersionNotFound.to_u32(),
},
],
);
assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound)));
assert!(matches!(
results.get(1).expect("slot 1 should exist"),
Err(DiskError::FileVersionNotFound)
));
}
#[test]
fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() {
let expected_items = vec![BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
}];
let mismatched = map_batch_read_version_responses(
&expected_items,
vec![BatchReadVersionResp {
index: 0,
path: "object-b".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-b".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
}],
)
.pop()
.expect("slot 0 should exist")
.expect_err("identity mismatch should fail closed");
assert!(
mismatched.to_string().contains("identity mismatch"),
"unexpected mismatch error: {mismatched}"
);
let duplicate = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
],
)
.pop()
.expect("slot 0 should exist")
.expect_err("duplicate response index should fail closed");
assert!(
duplicate.to_string().contains("duplicate index"),
"unexpected duplicate error: {duplicate}"
);
}
/// Isolation guard: unobserved objects record nothing (so parallel tests do
/// not inflate one another), and a scope clears its own counts on drop.
#[tokio::test]
crates/ecstore/src/set_disk/mod.rs
+30 -11
View File
@@ -97,7 +97,7 @@ use crate::storage_api_contracts::{
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
},
namespace::NamespaceLocking as _,
object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
range::HTTPRangeSpec,
};
use crate::store::utils::is_reserved_or_invalid_bucket;
@@ -735,8 +735,12 @@ pub(crate) use core::io_primitives::disk_call_counters;
mod ctx;
mod metadata;
mod ops;
#[cfg(test)]
pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
#[cfg(any(test, feature = "test-util"))]
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(test)]
pub(crate) use ops::object::DeleteObjectCommitBarrier;
#[cfg(feature = "test-util")]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
pub(crate) use ops::object::body_cache_plaintext_len;
@@ -922,14 +926,10 @@ mod prepared_get_object_metadata_tests {
.expect("test should find an object whose initial fanout covers both data shards")
}
#[allow(
dead_code,
reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)"
)]
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
fn bounded_initial_parity_disk_index(bucket: &str, object: &str) -> usize {
*bounded_metadata_fanout_order(bucket, object, 4, 2)
.get(3)
.expect("4-disk test geometry should leave one bounded spare disk")
.get(2)
.expect("4-disk test geometry should schedule one parity disk initially")
}
#[tokio::test]
@@ -1087,7 +1087,7 @@ mod prepared_get_object_metadata_tests {
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
],
async {
let slow_parity_disk = bounded_spare_disk_index(bucket, &object);
let slow_parity_disk = bounded_initial_parity_disk_index(bucket, &object);
let barrier =
rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION);
let calls = disk_call_counters::observe(&object);
@@ -3029,6 +3029,16 @@ pub struct SetDisks {
storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
}
// DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones
// each endpoint's canonical Arc, so an exact Arc set identifies the lock domain.
pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
left.iter()
.all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client)))
&& right
.iter()
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
}
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -3604,6 +3614,15 @@ impl SetDisks {
&self.ctx
}
/// Whether both sets' namespace-lock implementations cover the same object key.
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
(true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers),
_ => false,
}
}
/// The lock manager this set actually uses (test-only; Phase 5 Slice 3).
#[cfg(test)]
pub(crate) fn local_lock_manager_for_test(&self) -> &Arc<rustfs_lock::GlobalLockManager> {
@@ -4588,11 +4607,11 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool {
}
fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool {
opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement)
opts.delete_marker || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.data_movement)
}
fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none());
let mut mark_delete = goi.version_id.is_some() || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none());
let mut delete_marker = opts.versioned;
if opts.version_id.is_some() {
crates/ecstore/src/set_disk/ops/multipart.rs
+97 -2
View File
@@ -32,6 +32,8 @@ use crate::crash_inject::{self, CrashPoint};
use crate::multipart_listing::paginate_multipart_listing;
use futures::{StreamExt, stream};
use std::future::Future;
#[cfg(test)]
use std::sync::atomic::AtomicBool;
#[cfg(any(test, feature = "test-util"))]
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
@@ -65,6 +67,7 @@ impl StaleMultipartCleanupGuard {
#[cfg(any(test, feature = "test-util"))]
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum MultipartCommitPause {
NewUploadBeforeLockLost,
PutPartBeforeLockAcquire,
PutPartBeforeLockLost,
PutPartAfterRename,
@@ -156,6 +159,72 @@ impl Drop for MultipartCommitBarrier {
}
}
#[cfg(test)]
struct NewMultipartUploadCommitObservationState {
bucket: String,
object: String,
committed: AtomicBool,
}
#[cfg(test)]
pub(crate) struct NewMultipartUploadCommitObservation {
state: Arc<NewMultipartUploadCommitObservationState>,
}
#[cfg(test)]
static NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<NewMultipartUploadCommitObservationState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
impl NewMultipartUploadCommitObservation {
pub(crate) fn install(bucket: &str, object: &str) -> Self {
let state = Arc::new(NewMultipartUploadCommitObservationState {
bucket: bucket.to_string(),
object: object.to_string(),
committed: AtomicBool::new(false),
});
let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("new multipart upload commit observation mutex should not poison");
assert!(slot.is_none(), "new multipart upload commit observation must be unique");
*slot = Some(Arc::clone(&state));
Self { state }
}
pub(crate) fn committed(&self) -> bool {
self.state.committed.load(Ordering::Acquire)
}
}
#[cfg(test)]
impl Drop for NewMultipartUploadCommitObservation {
fn drop(&mut self) {
let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("new multipart upload commit observation mutex should not poison");
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
*slot = None;
}
}
}
#[cfg(test)]
fn observe_new_multipart_upload_commit(bucket: &str, object: &str) {
let state = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("new multipart upload commit observation mutex should not poison")
.as_ref()
.filter(|state| state.bucket == bucket && state.object == object)
.cloned();
if let Some(state) = state {
state.committed.store(true, Ordering::Release);
}
}
#[cfg(any(test, feature = "test-util"))]
async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
let barrier = {
@@ -1575,7 +1644,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let parts_metadata = vec![fi.clone(); disks.len()];
if !user_defined.contains_key("content-type") {
// TODO: get content-type
// TODO(backlog): detect content-type from part data when header is missing
}
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
@@ -1615,6 +1684,30 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement);
#[cfg(any(test, feature = "test-util"))]
pause_multipart_commit(bucket, object, MultipartCommitPause::NewUploadBeforeLockLost).await;
if _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "new_multipart_upload_commit",
bucket: bucket.to_string(),
object: object.to_string(),
required: 1,
achieved: 0,
});
}
if opts
.namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "new_multipart_upload_outer_lock",
bucket: bucket.to_string(),
object: object.to_string(),
required: 1,
achieved: 0,
});
}
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
Self::write_unique_file_info(
&shuffle_disks,
@@ -1626,6 +1719,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
)
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
#[cfg(test)]
observe_new_multipart_upload_commit(bucket, object);
// evalDisks
@@ -1971,7 +2066,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
}
// TODO: crypto
// TODO(backlog): integrate encryption verification during complete multipart
if (i < uploaded_parts.len() - 1)
&& !(opts.data_movement && ext_part.actual_size < 0)
crates/ecstore/src/set_disk/ops/object.rs
+195 -15
View File
@@ -45,6 +45,7 @@ use crate::bucket::replication::{
DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
};
use crate::data_usage::quota_object_size;
use crate::diagnostics::get::GetObjectFailureReason;
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
use crate::error::is_err_invalid_upload_id;
@@ -1293,7 +1294,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
(prepared.snapshot, prepared.object_info)
} else {
match self
.get_object_fileinfo(
.get_object_fileinfo_for_get_object_reader(
bucket,
object,
opts,
@@ -2122,11 +2123,11 @@ impl SetDisks {
let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
let put_object_size = known_put_object_storage_size(data.size());
let is_inline_buffer =
storage_class_config.should_inline(erasure.shard_file_size(put_object_size), erasure.data_shards, opts.versioned);
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
let shard_file_size = erasure.shard_file_size(put_object_size);
let shard_file_size = shard_file_size_raw;
let shard_size = erasure.shard_size();
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
@@ -2483,6 +2484,7 @@ impl SetDisks {
})
.await?,
);
notify_put_object_commit_namespace_acquired(bucket, object);
}
#[cfg(not(any(test, feature = "test-util")))]
{
@@ -4630,6 +4632,7 @@ struct PutObjectCommitBarrierState {
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
namespace_pending: tokio::sync::Notify,
namespace_acquired: std::sync::atomic::AtomicBool,
}
#[cfg(any(test, feature = "test-util"))]
@@ -4651,6 +4654,7 @@ impl PutObjectCommitBarrier {
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Notify::new(),
namespace_pending: tokio::sync::Notify::new(),
namespace_acquired: std::sync::atomic::AtomicBool::new(false),
});
let mut slot = PUT_OBJECT_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
@@ -4685,6 +4689,10 @@ impl PutObjectCommitBarrier {
.await
.expect("put object should wait for the namespace lock after leaving the commit barrier");
}
pub fn namespace_acquired(&self) -> bool {
self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
}
}
#[cfg(any(test, feature = "test-util"))]
@@ -4741,6 +4749,22 @@ fn notify_put_object_commit_namespace_pending(bucket: &str, object: &str) {
}
}
#[cfg(any(test, feature = "test-util"))]
fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
let barrier = PUT_OBJECT_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("put object commit barrier mutex should not poison")
.iter()
.find(|barrier| {
barrier.bucket == bucket && barrier.object == object && barrier.pause == PutObjectCommitPause::BeforeNamespace
})
.cloned();
if let Some(barrier) = barrier {
barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
}
}
#[cfg(test)]
struct DeleteObjectCommitBarrierState {
bucket: String,
@@ -4750,7 +4774,7 @@ struct DeleteObjectCommitBarrierState {
}
#[cfg(test)]
struct DeleteObjectCommitBarrier {
pub(crate) struct DeleteObjectCommitBarrier {
state: Arc<DeleteObjectCommitBarrierState>,
}
@@ -4760,7 +4784,7 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
#[cfg(test)]
impl DeleteObjectCommitBarrier {
fn install(bucket: &str, object: &str) -> Self {
pub(crate) fn install(bucket: &str, object: &str) -> Self {
let state = Arc::new(DeleteObjectCommitBarrierState {
bucket: bucket.to_string(),
object: object.to_string(),
@@ -4776,13 +4800,13 @@ impl DeleteObjectCommitBarrier {
Self { state }
}
async fn wait_until_paused(&self) {
pub(crate) async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("delete object should reach the deterministic commit barrier");
}
fn release(&self) {
pub(crate) fn release(&self) {
self.state.release.notify_one();
}
}
@@ -5655,7 +5679,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
(deleted, errors)
}
async fn delete_objects_with_accounting(
&self,
bucket: &str,
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut accounting = vec![None; objects.len()];
let delete_config_snapshot = opts
.delete_replication_config_snapshot
.clone()
@@ -5745,7 +5780,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
*item = Some(Error::other(message.clone()));
}
}
return (del_objects, del_errs);
return (del_objects, del_errs, accounting);
}
},
}
@@ -5792,6 +5827,22 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let source_missing = gerr
.as_ref()
.is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
// Resolve accounting from the generation selected under this
// object's write lock. A request-layer pre-stat is only an
// optimization and cannot identify a concurrent overwrite.
let (accounting_size, accounting_version_id, removed_current_object) = if source_missing
|| dobj.synthetic_version_id
|| set_disk_delete_creates_delete_marker(&check_opts)
|| goi.delete_marker
{
(None, None, false)
} else {
(
quota_object_size(&goi).ok(),
goi.version_id.filter(|version_id| !version_id.is_nil()),
(dobj.version_id.is_none() || is_explicit_null_version(dobj.version_id)) && !dobj.synthetic_version_id,
)
};
// Normalize both sides before comparing. `goi.version_id` is the
// client-facing identity, where `from_file_info` synthesizes
// `Some(Uuid::nil())` for a null version on a versioned or
@@ -5877,6 +5928,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
if dobj.version_id.is_none() && (version_suspended || versioned) {
vr.mod_time = Some(OffsetDateTime::now_utc());
vr.deleted = true;
vr.mark_deleted = true;
if versioned {
vr.version_id = Some(Uuid::new_v4());
}
@@ -5920,7 +5972,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
},
replication_state: vr.replication_state_internal.clone(),
..Default::default()
}
};
accounting[i] = Some(DeleteAccounting {
size: accounting_size,
version_id: accounting_version_id,
removed_current_object,
});
}
// Only add to vers_map if we hold the lock
@@ -5966,7 +6023,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
});
}
}
return (del_objects, del_errs);
return (del_objects, del_errs, accounting);
}
let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len());
@@ -6161,7 +6218,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
join_all(rollback_futures).await;
// TODO: add_partial
// TODO(backlog): support partial object deletion for multi-part objects
if let Some(api) = opts.tier_delete_journal_api.as_ref() {
for (idx, je) in persisted_journal_entries {
@@ -6204,7 +6261,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
}
(del_objects, del_errs)
// An accounting identity is actionable only when the delete result is
// successful. Never let a failed commit (including a partial quorum
// failure) reach the request-layer fast delta path.
for (index, err) in del_errs.iter().enumerate() {
if err.is_some() {
accounting[index] = None;
}
}
(del_objects, del_errs, accounting)
}
#[tracing::instrument(skip(self))]
@@ -6371,7 +6437,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
}
// TODO: Lifecycle
// TODO(backlog): integrate lifecycle evaluation before object deletion
let mut version_found = true;
// delete_object_version below derives its own majority quorum from the
@@ -6465,7 +6531,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
mark_deleted: mark_delete,
mod_time: Some(mod_time),
replication_state_internal: opts.delete_replication.as_ref().map(replication_state_to_filemeta),
..Default::default() // TODO: Transition
..Default::default() // TODO(backlog): populate transition state on delete markers
};
fi.set_tier_free_version_id(&find_vid.to_string());
@@ -6533,6 +6599,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
obj_info.size = goi.size;
// Keep the committed source metadata on the internal delete result so
// the request layer can derive canonical accounting for this exact
// generation. Delete responses do not expose these fields.
obj_info.actual_size = goi.actual_size;
obj_info.user_defined = Arc::clone(&goi.user_defined);
obj_info.parts = Arc::clone(&goi.parts);
obj_info.user_tags = Arc::clone(&goi.user_tags);
self.invalidate_get_object_metadata_cache(bucket, object).await;
Ok(obj_info)
@@ -7824,6 +7896,113 @@ mod replication_quota_safety_tests {
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
}
#[tokio::test]
async fn delete_returns_canonical_compressed_accounting_size() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
let bucket = "compressed-delete-accounting";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut user_defined = HashMap::new();
insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
.expect("compressed fixture reader should be valid"),
);
set_disks
.put_object(
bucket,
"object",
&mut reader,
&ObjectOptions {
user_defined,
..Default::default()
},
)
.await
.expect("compressed object should be written");
let (deleted, errors, accounting) = set_disks
.delete_objects_with_accounting(
bucket,
vec![ObjectToDelete {
object_name: "object".to_string(),
..Default::default()
}],
ObjectOptions {
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
)
.await;
assert!(errors[0].is_none(), "compressed delete should succeed: {:?}", errors[0]);
assert!(deleted[0].found, "the committed object must be reported as found");
assert_eq!(accounting[0].as_ref().and_then(|value| value.size), Some(1000));
assert!(accounting[0].as_ref().is_some_and(|value| value.version_id.is_none()));
assert!(accounting[0].as_ref().is_some_and(|value| value.removed_current_object));
}
#[tokio::test]
async fn suspended_delete_marker_does_not_return_body_accounting() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
let bucket = "suspended-delete-accounting";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut user_defined = HashMap::new();
insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
.expect("compressed fixture reader should be valid"),
);
let suspended_opts = ObjectOptions {
version_suspended: true,
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
s3s::dto::VersioningConfiguration {
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
..Default::default()
},
None,
))),
user_defined,
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
..Default::default()
};
set_disks
.put_object(bucket, "object", &mut reader, &suspended_opts)
.await
.expect("compressed object should be written");
let (deleted, errors, accounting) = set_disks
.delete_objects_with_accounting(
bucket,
vec![ObjectToDelete {
object_name: "object".to_string(),
..Default::default()
}],
suspended_opts,
)
.await;
assert!(errors[0].is_none(), "suspended delete should create a marker: {:?}", errors[0]);
assert!(deleted[0].delete_marker);
assert!(accounting[0].is_none(), "a delete marker must not carry body accounting");
}
#[tokio::test]
async fn direct_put_cannot_persist_a_tiny_logical_size() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
@@ -11636,6 +11815,7 @@ mod transition_upload_integrity_tests {
crate::data_movement::SourceCleanupBucketFence {
expected_incarnation_id: None,
lifecycle_guard: Some(&bucket_guard),
..Default::default()
},
"test_data_movement",
)
crates/ecstore/src/set_disk/read.rs
+66 -14
View File
@@ -259,10 +259,33 @@ impl SetDisks {
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false)
.await
}
#[tracing::instrument(level = "debug", skip(self))]
#[hotpath::measure(impl_type = "SetDisks")]
pub(super) async fn get_object_fileinfo_for_get_object_reader(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<GetObjectFileInfo> {
let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket)
&& crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
self.get_object_fileinfo_gated_inner(
bucket,
object,
opts,
read_data,
caller_allows_early_stop,
allow_read_version_coalescing,
)
.await
}
/// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
/// quorum fanout. Read-before-write callers (object tagging) must use this:
/// the returned online-disk set is the write target, and the early-stop
@@ -275,6 +298,20 @@ impl SetDisks {
opts: &ObjectOptions,
read_data: bool,
allow_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false)
.await
}
#[allow(clippy::too_many_arguments)]
async fn get_object_fileinfo_gated_inner(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
read_data: bool,
allow_early_stop: bool,
allow_read_version_coalescing: bool,
) -> Result<GetObjectFileInfo> {
let vid = opts.version_id.clone().unwrap_or_default();
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
@@ -337,19 +374,34 @@ impl SetDisks {
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
// core/io_primitives.rs); unsafe requests and callers that opt out
// (allow_early_stop=false) fall back to full-wait.
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
false,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?;
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing {
Self::read_all_fileinfo_observed_for_get_object(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?
} else {
Self::read_all_fileinfo_observed(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
false,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?
};
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
GET_OBJECT_PATH_INTERNAL_META
} else {
crates/ecstore/src/storage_api_contracts/mod.rs
+2 -2
View File
@@ -62,8 +62,8 @@ pub(crate) mod object {
use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::range::HTTPRangeSpec;
pub(crate) use rustfs_storage_api::{
DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, ObjectOperations,
ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions,
ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
};
pub(crate) trait EcstoreObjectIO:
crates/ecstore/src/store/bucket.rs
+1 -1
View File
@@ -601,7 +601,7 @@ impl ECStore {
#[instrument(skip(self))]
pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
// TODO: opts.cached
// TODO(backlog): support cached bucket listing via opts.cached
let mut buckets = self.peer_sys.list_bucket(opts).await?;
crates/ecstore/src/store/heal.rs
+608 -5
View File
@@ -13,7 +13,12 @@
// limitations under the License.
use super::*;
use crate::core::pools::POOL_META_NAME;
use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
use crate::set_disk::get_lock_acquire_timeout;
use crate::storage_api_contracts::heal::HealOperations as _;
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use rustfs_lock::NamespaceLockGuard;
use tracing::trace;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -30,7 +35,119 @@ fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
)
}
#[derive(Debug, Clone, Copy)]
enum HealFormatPoolSkip {
Completed,
Retryable,
}
fn classify_heal_format_pool(
pool_idx: usize,
pool_cmd_line: &str,
pool_meta: &PoolMeta,
rebalance_meta: Option<&RebalanceMeta>,
) -> Option<HealFormatPoolSkip> {
let Some(pool) = pool_meta.pools.get(pool_idx) else {
return Some(HealFormatPoolSkip::Retryable);
};
if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line {
return Some(HealFormatPoolSkip::Retryable);
}
if let Some(decommission) = pool.decommission.as_ref() {
if decommission.complete {
return Some(HealFormatPoolSkip::Completed);
}
if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) {
return Some(HealFormatPoolSkip::Retryable);
}
}
if let Some(meta) = rebalance_meta {
let Some(pool_stats) = meta.pool_stats.get(pool_idx) else {
return Some(HealFormatPoolSkip::Retryable);
};
if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) {
return Some(HealFormatPoolSkip::Retryable);
}
}
None
}
fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error {
match skip {
HealFormatPoolSkip::Completed => StorageError::NoHealRequired,
HealFormatPoolSkip::Retryable => StorageError::SlowDown,
}
}
fn heal_format_fence_lost_error() -> Error {
StorageError::SlowDown
}
impl ECStore {
async fn acquire_heal_format_fence(
&self,
) -> Result<(NamespaceLockGuard, NamespaceLockGuard, PoolMeta, Option<RebalanceMeta>)> {
let metadata_pool = self
.pools
.first()
.cloned()
.ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
// Metadata fence order is part of the decommission/rebalance protocol:
// pool.bin must always be acquired before rebalance.bin.
let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
let pool_guard = pool_lock.get_write_lock(get_lock_acquire_timeout()).await?;
let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?;
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
return Err(heal_format_fence_lost_error());
}
let mut pool_meta = PoolMeta::default();
pool_meta.load_no_lock(metadata_pool.clone()).await?;
if pool_meta.pools.len() != self.pools.len()
|| pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line
})
{
return Err(heal_format_fence_lost_error());
}
let mut rebalance_meta = RebalanceMeta::new();
let rebalance_meta = match rebalance_meta
.load_with_opts(
metadata_pool,
ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
{
Ok(()) => Some(rebalance_meta),
Err(Error::ConfigNotFound) => None,
Err(err) => return Err(err),
};
if rebalance_meta
.as_ref()
.is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
{
return Err(heal_format_fence_lost_error());
}
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
return Err(heal_format_fence_lost_error());
}
Ok((pool_guard, rebalance_guard, pool_meta, rebalance_meta))
}
fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
match opts.pool {
Some(pool_idx) => Ok(vec![
@@ -52,9 +169,26 @@ impl ECStore {
};
let mut count_no_heal = 0;
let mut count_completed = 0;
let mut first_error = None;
for pool in self.pools.iter() {
let (mut result, err) = pool.heal_format(dry_run).await?;
for (pool_idx, pool) in self.pools.iter().enumerate() {
let (pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
first_error.get_or_insert(heal_format_fence_lost_error());
break;
}
if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
{
if matches!(skip, HealFormatPoolSkip::Completed) {
count_completed += 1;
} else {
first_error.get_or_insert(heal_format_pool_skip_error(skip));
}
continue;
}
let fence_lost = || pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost();
let (mut result, err) = pool.heal_format_with_fence(dry_run, fence_lost).await?;
if let Some(err) = err {
match err {
StorageError::NoHealRequired => {
@@ -69,11 +203,18 @@ impl ECStore {
r.set_count += result.set_count;
r.before.drives.append(&mut result.before.drives);
r.after.drives.append(&mut result.after.drives);
// A lease can be lost after the final write; fail closed before
// reporting the pool as successfully healed.
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
first_error.get_or_insert(heal_format_fence_lost_error());
break;
}
}
if let Some(err) = first_error {
return Ok((r, Some(err)));
}
if count_no_heal == self.pools.len() {
if count_no_heal + count_completed == self.pools.len() {
info!(
event = EVENT_HEAL_FORMAT_COMPLETED,
component = LOG_COMPONENT_ECSTORE,
@@ -297,10 +438,17 @@ impl ECStore {
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::metadata_sys;
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
use crate::disk::{DiskOption, format::FormatV3, new_disk};
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk};
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::runtime::instance::InstanceContext;
use crate::services::rebalance::{RebalanceInfo, RebalanceStats};
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
use crate::store::init_format::{load_format_erasure, save_format_file};
use crate::store::init_local_disks_with_instance_ctx;
use tokio_util::sync::CancellationToken;
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
let format = FormatV3::new(1, 1);
@@ -347,6 +495,209 @@ mod tests {
}
}
fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
PoolMeta {
pools: vec![PoolStatus {
id: 0,
cmd_line: "pool-0".to_string(),
last_update: OffsetDateTime::UNIX_EPOCH,
decommission: Some(info),
}],
..Default::default()
}
}
#[test]
fn heal_format_pool_state_barriers_are_classified() {
let active = pool_meta_with_decommission(PoolDecommissionInfo {
start_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
});
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &active, None),
Some(HealFormatPoolSkip::Retryable)
));
for info in [
PoolDecommissionInfo {
failed: true,
..Default::default()
},
PoolDecommissionInfo {
canceled: true,
..Default::default()
},
] {
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None),
Some(HealFormatPoolSkip::Retryable)
));
}
let completed = pool_meta_with_decommission(PoolDecommissionInfo {
complete: true,
..Default::default()
});
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &completed, None),
Some(HealFormatPoolSkip::Completed)
));
}
#[test]
fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() {
let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default());
let rebalance = RebalanceMeta {
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)),
Some(HealFormatPoolSkip::Retryable)
));
let stopping = RebalanceMeta {
pool_stats: vec![RebalanceStats {
info: RebalanceInfo {
stopping: true,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)),
Some(HealFormatPoolSkip::Retryable)
));
let identity = pool_meta_with_decommission(PoolDecommissionInfo::default());
assert!(matches!(
classify_heal_format_pool(0, "pool-new", &identity, None),
Some(HealFormatPoolSkip::Retryable)
));
let identity_without_decommission = PoolMeta {
pools: vec![PoolStatus {
id: 0,
cmd_line: "pool-0".to_string(),
last_update: OffsetDateTime::UNIX_EPOCH,
decommission: None,
}],
..Default::default()
};
assert!(matches!(
classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None),
Some(HealFormatPoolSkip::Retryable)
));
assert!(matches!(
classify_heal_format_pool(0, "", &identity_meta, None),
Some(HealFormatPoolSkip::Retryable)
));
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None),
Some(HealFormatPoolSkip::Retryable)
));
let stopped = RebalanceMeta {
stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Stopped,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none());
let stopping_after_stop = RebalanceMeta {
stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Started,
stopping: true,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)),
Some(HealFormatPoolSkip::Retryable)
));
}
#[test]
fn skipped_heal_format_pool_is_never_reported_as_success() {
assert!(matches!(
heal_format_pool_skip_error(HealFormatPoolSkip::Retryable),
StorageError::SlowDown
));
assert!(matches!(
heal_format_pool_skip_error(HealFormatPoolSkip::Completed),
StorageError::NoHealRequired
));
}
async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
let mut pool_endpoints = Vec::new();
for pool_index in 0..2 {
let mut endpoints = Vec::new();
for disk_index in 0..4 {
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
tokio::fs::create_dir_all(&disk_path)
.await
.expect("multi-pool heal test disk should be created");
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
.expect("test endpoint should parse");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
}
pool_endpoints.push(PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("heal-owner-pool-{pool_index}"),
platform: "test".to_string(),
});
}
let endpoint_pools = EndpointServerPools::from(pool_endpoints);
let instance_ctx = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("multi-pool local disks should initialize");
let shutdown = CancellationToken::new();
let store = ECStore::new_with_instance_ctx(
"127.0.0.1:0".parse().expect("test address should parse"),
endpoint_pools,
shutdown.clone(),
instance_ctx,
)
.await
.expect("multi-pool test store should initialize");
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
(temp_dir, store, shutdown)
}
#[tokio::test]
async fn heal_object_pool_scope_selects_only_requested_pool() {
let store = minimal_heal_store().await;
@@ -506,6 +857,229 @@ mod tests {
}
}
#[tokio::test]
#[serial_test::serial]
async fn unscoped_heal_object_suspended_owner_semantics() {
let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
let active_object = "active-owner";
let suspended_only_object = "suspended-only";
let duplicate_object = "duplicate-owner";
let marker_object = "marker-owner";
let quorum_object = "quorum-owner";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in all pools");
let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
store.pools[0]
.put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
.await
.expect("active owner object should be written");
let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
missing_active_disk
.delete(
&bucket,
active_object,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("active owner shard should be removed for repair");
assert!(
missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
"the active owner fixture must start with one missing metadata copy"
);
let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
store.pools[1]
.put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
.await
.expect("suspended owner object should be written");
for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
store.pools[pool_index]
.put_object(
&bucket,
duplicate_object,
&mut duplicate_reader,
&ObjectOptions {
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
..Default::default()
},
)
.await
.expect("duplicate owner object should be written");
}
let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
.clone()
.expect("duplicate active owner disk should be online");
duplicate_missing_disk
.delete(
&bucket,
duplicate_object,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("duplicate active owner shard should be removed for repair");
let history_version = Uuid::new_v4();
let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
store.pools[0]
.put_object(
&bucket,
marker_object,
&mut history_reader,
&ObjectOptions {
versioned: true,
version_id: Some(history_version.to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
..Default::default()
},
)
.await
.expect("versioned marker history should be written");
store.pools[0]
.delete_object(
&bucket,
marker_object,
ObjectOptions {
versioned: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
..Default::default()
},
)
.await
.expect("delete marker should be written");
let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
store.pools[0]
.put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
.await
.expect("quorum boundary object should be written");
{
let mut pool_meta = store.pool_meta.write().await;
let mut next = PoolMeta::new(&store.pools, &pool_meta);
next.pools[1].decommission = Some(PoolDecommissionInfo {
start_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
});
*pool_meta = next;
}
let (_, duplicate_owner) = store
.get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
.await
.expect("duplicate owner should resolve");
assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
let (_, active_duplicate_owner) = store
.get_latest_object_info_with_idx(
&bucket,
duplicate_object,
&ObjectOptions {
skip_decommissioned: true,
..Default::default()
},
)
.await
.expect("active duplicate owner should resolve");
assert_eq!(
active_duplicate_owner, 0,
"suspended duplicate must be excluded from active owner selection"
);
let (duplicate_result, duplicate_err) = store
.handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
.await
.expect("duplicate owner heal should complete through the production path");
assert_eq!(duplicate_result.object, duplicate_object);
assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
assert!(
duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
"production heal must repair the active duplicate owner rather than the suspended owner"
);
let (marker_info, marker_owner) = store
.get_latest_object_info_with_idx(
&bucket,
marker_object,
&ObjectOptions {
skip_decommissioned: true,
versioned: true,
..Default::default()
},
)
.await
.expect("latest delete marker should resolve");
assert_eq!(marker_owner, 0);
assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
let (active_result, active_err) = store
.handle_heal_object(&bucket, active_object, "", &HealOpts::default())
.await
.expect("unscoped active-owner heal should complete");
assert_eq!(active_result.object, active_object);
assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
assert!(
missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
"active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
);
assert!(
store.pools[1]
.get_object_info(&bucket, active_object, &ObjectOptions::default())
.await
.is_err(),
"the suspended pool must not be written for an active-owner object"
);
let (suspended_result, suspended_err) = store
.handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
.await
.expect("unscoped suspended-only heal should return a terminal result");
assert!(suspended_result.object.is_empty());
assert!(matches!(suspended_err, Some(Error::FileNotFound)));
assert!(
store.pools[1]
.get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
.await
.is_ok(),
"suspended-only data must remain untouched when unscoped heal reports absent"
);
let (_, explicit_err) = store
.handle_heal_object(
&bucket,
suspended_only_object,
"",
&HealOpts {
pool: Some(1),
..Default::default()
},
)
.await
.expect("explicit suspended-owner heal should return a mapped error");
assert!(matches!(explicit_err, Some(Error::SlowDown)));
let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
let surviving_quorum_disk = original_quorum_disks[3].clone();
*store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
let (_, quorum_err) = store
.handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
.await
.expect("quorum boundary heal should return a mapped result");
*store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
assert!(
matches!(quorum_err, Some(Error::ErasureReadQuorum)),
"quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
);
shutdown.cancel();
}
#[tokio::test]
async fn handle_heal_format_continues_after_a_pool_error() {
let canonical_format = FormatV3::new(1, 3);
@@ -615,6 +1189,18 @@ mod tests {
bucket_fence_registry: std::sync::Arc::default(),
};
let err = store
.handle_heal_format(false)
.await
.expect_err("missing pool metadata must fail closed before format writes");
assert!(matches!(err, StorageError::SlowDown));
let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
pool_meta
.save(store.pools.clone())
.await
.expect("pool metadata should be persisted before format heal");
let (result, err) = store
.handle_heal_format(false)
.await
@@ -628,5 +1214,22 @@ mod tests {
.await
.expect("the later pool should be healed despite the first pool error");
assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
for status in &mut completed_meta.pools {
status.decommission = Some(PoolDecommissionInfo {
complete: true,
..Default::default()
});
}
completed_meta
.save(store.pools.clone())
.await
.expect("completed pool metadata should be persisted");
let (_, err) = store
.handle_heal_format(false)
.await
.expect("completed pools should be reported as a no-op");
assert!(matches!(err, Some(StorageError::NoHealRequired)));
}
}
crates/ecstore/src/store/init.rs
+1352 -2
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/store/list_objects.rs
+2 -2
View File
@@ -4673,7 +4673,7 @@ async fn gather_results(
entry.name = entry.name.replace("\\", "/");
}
// TODO: rx.recv()
// TODO(backlog): integrate rx.recv() for incremental listing results
if let Some(marker) = &opts.marker
&& ((!opts.include_marker && &entry.name <= marker) || (opts.include_marker && &entry.name < marker))
@@ -4703,7 +4703,7 @@ async fn gather_results(
continue;
}
// TODO: Lifecycle
// TODO(backlog): integrate lifecycle evaluation during object listing
entries.push(Some(entry));
candidate_entries += 1;
crates/ecstore/src/store/mod.rs
+3 -3
View File
@@ -33,7 +33,7 @@ use crate::bucket::utils::check_put_object_part_args;
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
use crate::cluster::rpc::{RemoteClient, S3PeerSys};
use crate::config::storageclass;
use crate::core::pools::PoolMeta;
use crate::core::pools::{DecommissionCanceler, PoolMeta};
use crate::disk::endpoint::{Endpoint, EndpointType};
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
use crate::error::{Error, Result};
@@ -151,7 +151,7 @@ pub(crate) mod init_format;
pub(crate) mod list_objects;
mod multipart;
mod object;
pub(crate) use object::ObjectLockDiagGuard;
pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence};
pub use object::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
SnapshotConsistencyError,
@@ -176,7 +176,7 @@ pub struct ECStore {
// pub local_disks: Vec<DiskStore>,
pub pool_meta: RwLock<PoolMeta>,
pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
pub decommission_cancelers: RwLock<Vec<Option<CancellationToken>>>,
pub decommission_cancelers: RwLock<Vec<Option<DecommissionCanceler>>>,
/// Serializes rebalance/decommission start transitions.
///
/// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`,
crates/ecstore/src/store/multipart.rs
+22 -11
View File
@@ -332,7 +332,7 @@ impl ECStore {
let expected_incarnation_id = opts.expected_bucket_incarnation_id;
if request.prefix.is_empty() {
// TODO: return from cache
// TODO(backlog): return cached multipart listing when prefix is empty
}
if self.single_pool() {
@@ -400,7 +400,7 @@ impl ECStore {
object: &str,
opts: &ObjectOptions,
) -> Result<MultipartUploadResult> {
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts, None)
.await
.map(|(res, _, _)| res)
}
@@ -410,20 +410,22 @@ impl ECStore {
bucket: &str,
object: &str,
opts: &ObjectOptions,
mutation_fence: Option<&ObjectLockDiagGuard>,
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
check_new_multipart_args(bucket, object)?;
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let opts = &opts;
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if self.single_pool() {
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
.await;
return self.pools[0]
.new_multipart_upload(bucket, object, opts)
.new_multipart_upload(bucket, object, &opts)
.await
.map(|res| (res, 0, opts.expected_bucket_incarnation_id));
}
if opts.data_movement && opts.version_id.is_some() {
let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?;
let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?;
if idx == opts.src_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
@@ -431,7 +433,9 @@ impl ECStore {
opts.version_id.clone().unwrap_or_default(),
));
}
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
return Ok((res, idx, opts.expected_bucket_incarnation_id));
}
@@ -454,7 +458,9 @@ impl ECStore {
.await?;
if !res.uploads.is_empty() {
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
return Ok((res, idx, opts.expected_bucket_incarnation_id));
}
}
@@ -467,7 +473,9 @@ impl ECStore {
));
}
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
Ok((res, idx, opts.expected_bucket_incarnation_id))
}
@@ -610,7 +618,7 @@ impl ECStore {
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let opts = &opts;
// TODO: defer DeleteUploadID
// TODO(backlog): defer DeleteUploadID to background for faster abort response
if self.single_pool() {
return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;
@@ -704,13 +712,14 @@ impl ECStore {
pub(crate) async fn complete_multipart_upload_for_data_movement(
self: Arc<Self>,
target_pool_idx: usize,
target: (usize, Option<&ObjectLockDiagGuard>),
bucket: &str,
object: &str,
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let (target_pool_idx, mutation_fence) = target;
check_complete_multipart_args(bucket, object, upload_id)?;
if !opts.data_movement {
return Err(Error::other("targeted multipart completion requires data_movement options"));
@@ -739,6 +748,8 @@ impl ECStore {
snapshot.add_lock_fences(&mut opts);
opts.object_lock_config_snapshot = Some(snapshot);
}
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
.await;
#[cfg(test)]
pause_data_movement_multipart_before_selected_completion(bucket).await;
let pool = self
crates/ecstore/src/store/object.rs
+840 -42
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/store/rebalance.rs
+387 -2
View File
@@ -385,7 +385,7 @@ impl ECStore {
}
pub(super) async fn is_suspended(&self, idx: usize) -> bool {
// TODO: LOCK
// TODO(backlog): acquire pool metadata lock for consistent suspension check
let pool_meta = self.pool_meta.read().await;
@@ -859,6 +859,7 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
use crate::disk::error::DiskError;
use crate::layout::endpoint::Endpoint;
@@ -1423,6 +1424,14 @@ mod tests {
}
}
fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
ObjectInfo {
version_id: Some(version_id),
etag,
..object_info_with_mod_time(unix_ts, delete_marker)
}
}
#[test]
fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
let candidates = vec![
@@ -1446,7 +1455,7 @@ mod tests {
}
#[test]
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() {
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_mod_time(10, false)),
@@ -1466,6 +1475,382 @@ mod tests {
assert_eq!(idx, 1);
}
#[test]
fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 2,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 7,
err: None,
},
];
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect("equivalent replicas must resolve deterministically");
assert_eq!(idx, 7);
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("divergent version ids must not silently resolve to the higher pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("divergent etags must not silently resolve to the higher pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("a delete marker tied with a live version must not be masked by the pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
let err = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(left),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(right),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect_err("equal-time identity divergence must fail closed");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut data_dir = base.clone();
data_dir.data_dir = Some(Uuid::from_u128(2));
assert_equal_time_identity_conflict(base.clone(), data_dir);
let mut size = base.clone();
size.size = 1;
assert_equal_time_identity_conflict(base.clone(), size);
let mut actual_size = base.clone();
actual_size.actual_size = 1;
assert_equal_time_identity_conflict(base.clone(), actual_size);
let mut checksum = base.clone();
checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
assert_equal_time_identity_conflict(base.clone(), checksum);
let mut parts = base.clone();
parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
etag: "part-etag".to_string(),
number: 1,
size: 1,
..Default::default()
}]);
assert_equal_time_identity_conflict(base.clone(), parts);
let mut transition = base;
transition.transitioned_object.tier = "tier-a".to_string();
assert_equal_time_identity_conflict(
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
transition,
);
}
#[test]
fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let mut minio_alias = base.clone();
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"X-MINIO-INTERNAL-COMPRESSION".to_string(),
"zstd".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(rustfs_alias),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(minio_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("same-value internal aliases should resolve");
assert_eq!(idx, 1);
let mut dual_alias = base.clone();
dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
("x-minio-internal-compression".to_string(), "zstd".to_string()),
]));
let mut single_alias = base;
single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(dual_alias),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(single_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("dual-key and single-key internal metadata should resolve");
assert_eq!(idx, 1);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let mut minio_alias = base;
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-minio-internal-compression".to_string(),
"snappy".to_string(),
)]));
assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
}
#[test]
fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!(
"X-RUSTFS-INTERNAL-{}{suffix}",
suffix_prefix.to_uppercase(),
suffix = "arn:aws:s3:::Bucket"
),
"value".to_string(),
)]));
let mut minio_alias = base.clone();
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
"value".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(rustfs_alias.clone()),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(minio_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("dynamic internal aliases with the same target should resolve");
assert_eq!(idx, 1);
let mut different_target_case = base;
different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
"value".to_string(),
)]));
assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
}
}
#[test]
fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut first = base.clone();
first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
("x-minio-internal-compression".to_string(), "snappy".to_string()),
]));
let mut second = base;
second.user_defined = first.user_defined.clone();
assert_equal_time_identity_conflict(first, second);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut replication = base.clone();
replication.replication_status_internal = Some("PENDING".to_string());
replication.replication_status = ReplicationStatusType::Pending;
assert_equal_time_identity_conflict(base.clone(), replication);
let mut purge = base.clone();
purge.version_purge_status_internal = Some("PENDING".to_string());
purge.version_purge_status = VersionPurgeStatusType::Pending;
assert_equal_time_identity_conflict(base.clone(), purge);
let mut decision = base;
decision.replication_decision = "replicate".to_string();
assert_equal_time_identity_conflict(
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
decision,
);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
without_mod_time.mod_time = None;
let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
}
#[test]
fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
older.data_dir = Some(Uuid::from_u128(2));
let (info, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(latest),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(older),
idx: 9,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("older identity divergence must not affect the latest candidate");
assert_eq!(idx, 0);
assert_eq!(
info.mod_time,
Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
);
}
#[test]
fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: None,
idx: 1,
err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
];
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect("not-found pools must not block resolution of found candidates");
assert_eq!(idx, 0);
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
}
#[test]
fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
let err = resolve_latest_object_info_candidates(
crates/ecstore/src/store/rebalance/support.rs
+175 -23
View File
@@ -12,10 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::cmp::Ordering;
use std::collections::HashMap;
use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{ObjectInfo, ObjectOptions};
use rustfs_utils::http::metadata_compat::{
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
strip_internal_prefix_preserving_case,
};
use rustfs_utils::path::decode_dir_object;
use time::OffsetDateTime;
@@ -73,7 +77,7 @@ pub(super) fn resolve_rebalance_delete_from_all_pools_result(
object: &str,
) -> Result<ObjectInfo> {
result.map_err(|err| {
if err == Error::PreconditionFailed {
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
err
} else {
Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}"))
@@ -86,7 +90,7 @@ fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
}
fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
if err == Error::PreconditionFailed {
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
err
} else {
Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
@@ -137,37 +141,158 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p
))
}
fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option<OffsetDateTime> {
candidate
.info
.as_ref()
.map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
}
fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
left.transition_version_state == right.transition_version_state
&& left.transitioned_object.name == right.transitioned_object.name
&& left.transitioned_object.version_id == right.transitioned_object.version_id
&& left.transitioned_object.tier == right.transitioned_object.tier
&& left.transitioned_object.free_version == right.transitioned_object.free_version
&& left.transitioned_object.status == right.transitioned_object.status
}
#[derive(PartialEq, Eq)]
struct LatestUserDefinedIdentity {
internal: HashMap<String, String>,
other: HashMap<String, String>,
}
fn normalize_internal_identity_suffix(key: &str) -> Option<String> {
let suffix = strip_internal_prefix_preserving_case(key)?;
for dynamic_prefix in [
SUFFIX_REPLICATION_RESET_ARN_PREFIX,
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
] {
let prefix_len = dynamic_prefix.len();
if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..))
&& prefix.eq_ignore_ascii_case(dynamic_prefix)
{
return Some(format!("{dynamic_prefix}{remainder}"));
}
}
Some(suffix.to_lowercase())
}
fn normalize_user_defined_identity(user_defined: &HashMap<String, String>) -> Option<LatestUserDefinedIdentity> {
let mut identity = LatestUserDefinedIdentity {
internal: HashMap::with_capacity(user_defined.len()),
other: HashMap::with_capacity(user_defined.len()),
};
for (key, value) in user_defined {
if let Some(suffix) = normalize_internal_identity_suffix(key) {
if identity
.internal
.insert(suffix, value.clone())
.is_some_and(|previous| previous != *value)
{
return None;
}
} else {
identity.other.insert(key.clone(), value.clone());
}
}
Some(identity)
}
fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
match (
normalize_user_defined_identity(&left.user_defined),
normalize_user_defined_identity(&right.user_defined),
) {
(Some(left), Some(right)) => left == right,
_ => false,
}
}
/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
/// differ for the same object version while the selected winner still carries
/// the chosen pool's layout. `put_object_reader` is also intentionally
/// excluded because it is a transient request handle that `ObjectInfo::clone`
/// drops. Every other ObjectInfo field is part of the production-visible
/// identity and must agree before the pool index can provide a deterministic
/// tie-break.
fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
left.bucket == right.bucket
&& left.name == right.name
&& left.storage_class == right.storage_class
&& left.mod_time == right.mod_time
&& left.size == right.size
&& left.actual_size == right.actual_size
&& left.is_dir == right.is_dir
&& same_user_defined_identity(left, right)
&& left.user_tags == right.user_tags
&& left.version_id == right.version_id
&& left.data_dir == right.data_dir
&& left.delete_marker == right.delete_marker
&& same_transition_identity(left, right)
&& left.restore_ongoing == right.restore_ongoing
&& left.restore_expires == right.restore_expires
&& left.parts == right.parts
&& left.is_latest == right.is_latest
&& left.content_type == right.content_type
&& left.content_encoding == right.content_encoding
&& left.expires == right.expires
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.etag == right.etag
&& left.inlined == right.inlined
&& left.metadata_only == right.metadata_only
&& left.version_only == right.version_only
&& left.replication_status_internal == right.replication_status_internal
&& left.replication_status == right.replication_status
&& left.version_purge_status_internal == right.version_purge_status_internal
&& left.version_purge_status == right.version_purge_status
&& left.replication_decision == right.replication_decision
&& left.checksum == right.checksum
}
pub(super) fn resolve_latest_object_info_candidates(
mut candidates: Vec<LatestObjectInfoCandidate>,
candidates: Vec<LatestObjectInfoCandidate>,
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, usize)> {
candidates.sort_by(|a, b| {
let a_mod = if let Some(info) = &a.info {
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
OffsetDateTime::UNIX_EPOCH
let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max();
if let Some(latest_mod_time) = latest_mod_time {
let mut latest_candidates = candidates
.into_iter()
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
.collect::<Vec<_>>();
latest_candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.idx));
let Some(winner) = latest_candidates.first() else {
return Err(Error::ErasureReadQuorum);
};
let Some(winner_info) = winner.info.as_ref() else {
return Err(Error::ErasureReadQuorum);
};
let b_mod = if let Some(info) = &b.info {
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
OffsetDateTime::UNIX_EPOCH
};
if a_mod == b_mod {
return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less };
if latest_candidates.iter().skip(1).any(|candidate| {
candidate
.info
.as_ref()
.is_none_or(|info| !same_latest_object_info_identity(winner_info, info))
}) {
return Err(Error::ErasureReadQuorum);
}
b_mod.cmp(&a_mod)
});
return Ok((winner_info.clone(), winner.idx));
}
for candidate in candidates {
if let Some(info) = candidate.info {
return Ok((info, candidate.idx));
}
if let Some(err) = candidate.err
&& !is_err_object_not_found(&err)
&& !is_err_version_not_found(&err)
@@ -191,6 +316,18 @@ mod tests {
assert_eq!(err, Error::PreconditionFailed);
}
#[test]
fn rebalance_delete_result_preserves_prefix_access_denied() {
let err = resolve_rebalance_delete_from_all_pools_result(
Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
"bucket",
"object",
)
.expect_err("prefix access denial should remain structured");
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
}
#[test]
fn rebalance_delete_pool_result_preserves_precondition_failed() {
let err = resolve_rebalance_delete_from_all_pools_results(
@@ -205,4 +342,19 @@ mod tests {
assert_eq!(err, Error::PreconditionFailed);
}
#[test]
fn rebalance_delete_pool_result_preserves_prefix_access_denied() {
let err = resolve_rebalance_delete_from_all_pools_results(
vec![RebalanceDeletePoolResult {
pool_idx: 0,
result: Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
}],
"bucket",
"object",
)
.expect_err("prefix access denial should remain structured");
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
}
}
crates/heal/src/heal/channel.rs
+54
View File
@@ -1640,6 +1640,60 @@ mod tests {
assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
}
#[tokio::test]
async fn test_process_query_request_reports_displaced_terminal_detail() {
let heal_manager = Arc::new(HealManager::new(
Arc::new(MockStorage),
Some(HealConfig {
queue_size: 1,
..HealConfig::default()
}),
));
let mut displaced = HealRequest::new(
HealType::Bucket {
bucket: "displaced-channel".to_string(),
},
HealOptions::default(),
HealPriority::Low,
);
displaced.id = "displaced-channel-task".to_string();
let displaced_id = displaced.id.clone();
heal_manager
.submit_heal_request(displaced)
.await
.expect("initial channel task should queue");
heal_manager
.submit_heal_request(HealRequest::new(
HealType::Bucket {
bucket: "successor-channel".to_string(),
},
HealOptions::default(),
HealPriority::High,
))
.await
.expect("successor channel task should displace the initial task");
let processor = HealChannelProcessor::new(heal_manager);
let (tx, rx) = oneshot::channel();
processor
.process_query_request("displaced-channel".to_string(), displaced_id, None, tx)
.await
.expect("displaced query should process");
let response = rx
.await
.expect("query response should be returned")
.expect("displaced query should remain successful");
let payload: serde_json::Value = serde_json::from_slice(response.data.as_deref().expect("status payload should exist"))
.expect("status payload should be json");
assert_eq!(payload["summary"], "stopped");
assert!(
response
.error
.as_deref()
.is_some_and(|detail| detail.contains("reason=displaced"))
);
}
#[tokio::test]
async fn test_process_query_request_reports_running_for_queued_task() {
let heal_manager = create_test_heal_manager();
crates/heal/src/heal/manager.rs
+105 -10
View File
@@ -40,6 +40,7 @@ use tracing::{debug, error, info, warn};
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again";
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
const LOG_SUBSYSTEM_MANAGER: &str = "manager";
@@ -120,26 +121,30 @@ struct MrfRepairNoticeTarget {
version_id: Option<[u8; 16]>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone)]
struct HealAdmissionDecision {
result: HealAdmissionResult,
displaced_task_id: Option<String>,
displaced_request: Option<HealRequest>,
}
impl HealAdmissionDecision {
const fn new(result: HealAdmissionResult) -> Self {
Self {
result,
displaced_task_id: None,
displaced_request: None,
}
}
fn accepted_with_displacement(displaced_task_id: String) -> Self {
fn accepted_with_displacement(displaced_request: HealRequest) -> Self {
Self {
result: HealAdmissionResult::Accepted,
displaced_task_id: Some(displaced_task_id),
displaced_request: Some(displaced_request),
}
}
fn displaced_task_id(&self) -> Option<&str> {
self.displaced_request.as_ref().map(|request| request.id.as_str())
}
}
fn lock_mrf_repair_notice_targets(
@@ -151,6 +156,55 @@ fn lock_mrf_repair_notice_targets(
}
}
fn lock_displaced_terminals(
registry: &StdMutex<HashMap<String, Arc<CompletedHealStatus>>>,
) -> StdMutexGuard<'_, HashMap<String, Arc<CompletedHealStatus>>> {
match registry.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
fn record_displaced_terminal(
registry: &StdMutex<HashMap<String, Arc<CompletedHealStatus>>>,
request: &HealRequest,
) -> Arc<CompletedHealStatus> {
let terminal = Arc::new(CompletedHealStatus {
heal_type: request.heal_type.clone(),
status: HealTaskStatus::Failed {
error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"),
},
result_items_truncated: false,
completed_at: SystemTime::now(),
seqed_items: Vec::new(),
next_seq: 0,
min_seq: 0,
});
let mut terminals = lock_displaced_terminals(registry);
prune_completed_heal_statuses(&mut terminals);
terminals.insert(request.id.clone(), Arc::clone(&terminal));
terminal
}
async fn remove_displaced_task_aliases(
aliases: &Arc<Mutex<HashMap<String, HealTaskAlias>>>,
terminals: &StdMutex<HashMap<String, Arc<CompletedHealStatus>>>,
task_id: &str,
terminal: &Arc<CompletedHealStatus>,
) {
let mut aliases = aliases.lock().await;
let alias_ids = aliases
.iter()
.filter_map(|(alias_id, alias)| (alias.task_id == task_id).then_some(alias_id.clone()))
.collect::<Vec<_>>();
let mut displaced_terminals = lock_displaced_terminals(terminals);
prune_completed_heal_statuses(&mut displaced_terminals);
for alias_id in alias_ids {
displaced_terminals.insert(alias_id, Arc::clone(terminal));
}
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
}
async fn remove_task_aliases_for_task(registry: &Arc<Mutex<HashMap<String, HealTaskAlias>>>, task_id: &str) {
registry
.lock()
@@ -618,6 +672,14 @@ pub struct HealManager {
/// are shared so the lookup helper can hand a completed entry to a
/// caller without cloning the retained result window.
completed_heals: Arc<Mutex<HashMap<String, Arc<CompletedHealStatus>>>>,
/// Terminals for requests removed by priority displacement. An Accepted
/// task ID remains queryable for the same process lifetime and the normal
/// ten-minute status TTL; clients should treat `reason=displaced` as a
/// terminal result and submit a fresh request. This sidecar is synchronous
/// so admission can publish the terminal while the queue transition is
/// still under its lock, without awaiting another tokio lock. Queue state
/// is process-local, so this guarantee does not extend across restart.
displaced_terminals: Arc<StdMutex<HashMap<String, Arc<CompletedHealStatus>>>>,
/// Client tokens merged into an existing task id.
task_aliases: Arc<Mutex<HashMap<String, HealTaskAlias>>>,
/// Heal tasks waiting for a retry backoff to expire.
@@ -659,6 +721,7 @@ struct HealQueueContext<'a> {
heal_queue: &'a Arc<Mutex<PriorityHealQueue>>,
active_heals: &'a Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
completed_heals: &'a Arc<Mutex<HashMap<String, Arc<CompletedHealStatus>>>>,
displaced_terminals: &'a Arc<StdMutex<HashMap<String, Arc<CompletedHealStatus>>>>,
task_aliases: &'a Arc<Mutex<HashMap<String, HealTaskAlias>>>,
retrying_heals: &'a Arc<Mutex<HashMap<String, RetryingHeal>>>,
mrf_repair_notice_targets: &'a Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>,
@@ -874,7 +937,7 @@ impl HealManager {
result = "accepted_by_displacement",
"Heal queue request accepted by displacement"
});
return HealAdmissionDecision::accepted_with_displacement(displaced.id);
return HealAdmissionDecision::accepted_with_displacement(displaced);
}
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
@@ -1105,6 +1168,7 @@ impl HealManager {
active_heals: Arc::new(Mutex::new(HashMap::new())),
heal_queue: Arc::new(Mutex::new(PriorityHealQueue::new())),
completed_heals: Arc::new(Mutex::new(HashMap::new())),
displaced_terminals: Arc::new(StdMutex::new(HashMap::new())),
task_aliases: Arc::new(Mutex::new(HashMap::new())),
retrying_heals: Arc::new(Mutex::new(HashMap::new())),
mrf_repair_notice_targets: Arc::new(StdMutex::new(HashMap::new())),
@@ -1209,6 +1273,10 @@ impl HealManager {
active_heals.clear();
publish_active_heal_count(&active_heals);
self.completed_heals.lock().await.clear();
// Do not let the synchronous guard live across the following async lock.
{
lock_displaced_terminals(&self.displaced_terminals).clear();
}
self.task_aliases.lock().await.clear();
self.retrying_heals.lock().await.clear();
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).clear();
@@ -1459,7 +1527,11 @@ impl HealManager {
task_id = queued_id.to_owned();
}
let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
let displaced_task_id = admission_decision.displaced_task_id;
let displaced_task_id = admission_decision.displaced_task_id().map(ToOwned::to_owned);
let displaced_terminal = admission_decision
.displaced_request
.as_ref()
.map(|request| record_displaced_terminal(&self.displaced_terminals, request));
if matches!(admission, HealAdmissionResult::Accepted | HealAdmissionResult::Merged)
&& let Some(target) = mrf_notice_target
{
@@ -1473,8 +1545,12 @@ impl HealManager {
drop(queue);
drop(active_heals);
if let Some(displaced_task_id) = displaced_task_id {
self.remove_aliases_for_task(&displaced_task_id).await;
if let (Some(displaced_task_id), Some(displaced_terminal)) = (displaced_task_id, displaced_terminal) {
// The queue has already removed the displaced request, so the
// synchronous terminal sidecar was published before aliases and
// MRF ownership are cleaned up.
remove_displaced_task_aliases(&self.task_aliases, &self.displaced_terminals, &displaced_task_id, &displaced_terminal)
.await;
}
if should_notify {
@@ -1549,6 +1625,15 @@ impl HealManager {
}
}
if terminal_completed.is_none() {
let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals);
prune_completed_heal_statuses(&mut displaced_terminals);
terminal_completed = displaced_terminals
.get(canonical_task_id)
.filter(|terminal| matches_path(&terminal.heal_type))
.cloned();
}
match terminal_completed {
Some(completed) => TaskStateLookup::Completed(completed),
None => TaskStateLookup::NotFound,
@@ -1669,9 +1754,19 @@ impl HealManager {
let mut completed_heals = self.completed_heals.lock().await;
prune_completed_heal_statuses(&mut completed_heals);
completed_heals
if completed_heals
.values()
.any(|completed| heal_type_matches_path(&completed.heal_type, heal_path))
{
return true;
}
drop(completed_heals);
let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals);
prune_completed_heal_statuses(&mut displaced_terminals);
displaced_terminals
.values()
.any(|terminal| heal_type_matches_path(&terminal.heal_type, heal_path))
}
/// Get task progress
crates/heal/src/heal/manager/auto_scan.rs
+15 -2
View File
@@ -21,6 +21,7 @@ impl HealManager {
let heal_queue = self.heal_queue.clone();
let active_heals = self.active_heals.clone();
let task_aliases = self.task_aliases.clone();
let displaced_terminals = self.displaced_terminals.clone();
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
let storage = self.storage.clone();
let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
@@ -481,6 +482,10 @@ impl HealManager {
let admission = admission_decision.result;
let should_notify =
matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
let displaced_terminal = admission_decision
.displaced_request
.as_ref()
.map(|request| record_displaced_terminal(&displaced_terminals, request));
if matches!(admission, HealAdmissionResult::Accepted)
&& let Some(anchor) = recovery_anchor
{
@@ -491,8 +496,16 @@ impl HealManager {
}
drop(queue);
drop(config);
if let Some(displaced_task_id) = admission_decision.displaced_task_id {
remove_task_aliases_for_task(&task_aliases, &displaced_task_id).await;
if let (Some(displaced_task_id), Some(displaced_terminal)) =
(admission_decision.displaced_task_id().map(ToOwned::to_owned), displaced_terminal)
{
remove_displaced_task_aliases(
&task_aliases,
&displaced_terminals,
&displaced_task_id,
&displaced_terminal,
)
.await;
lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id);
}
if matches!(admission, HealAdmissionResult::Accepted) {
crates/heal/src/heal/manager/scheduler.rs
+25 -3
View File
@@ -21,6 +21,7 @@ impl HealManager {
let heal_queue = self.heal_queue.clone();
let active_heals = self.active_heals.clone();
let completed_heals = self.completed_heals.clone();
let displaced_terminals = self.displaced_terminals.clone();
let task_aliases = self.task_aliases.clone();
let retrying_heals = self.retrying_heals.clone();
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
@@ -53,6 +54,7 @@ impl HealManager {
heal_queue: &heal_queue,
active_heals: &active_heals,
completed_heals: &completed_heals,
displaced_terminals: &displaced_terminals,
task_aliases: &task_aliases,
retrying_heals: &retrying_heals,
mrf_repair_notice_targets: &mrf_repair_notice_targets,
@@ -71,6 +73,7 @@ impl HealManager {
heal_queue: &heal_queue,
active_heals: &active_heals,
completed_heals: &completed_heals,
displaced_terminals: &displaced_terminals,
task_aliases: &task_aliases,
retrying_heals: &retrying_heals,
mrf_repair_notice_targets: &mrf_repair_notice_targets,
@@ -98,6 +101,7 @@ impl HealManager {
heal_queue,
active_heals,
completed_heals,
displaced_terminals,
task_aliases,
retrying_heals,
mrf_repair_notice_targets,
@@ -183,6 +187,7 @@ impl HealManager {
let active_heals_clone = active_heals.clone();
let heal_queue_clone = heal_queue.clone();
let completed_heals_clone = completed_heals.clone();
let displaced_terminals_clone = displaced_terminals.clone();
let task_aliases_clone = task_aliases.clone();
let retrying_heals_clone = retrying_heals.clone();
let mrf_repair_notice_targets_clone = mrf_repair_notice_targets.clone();
@@ -363,6 +368,7 @@ impl HealManager {
let retry_heal_queue = heal_queue_clone.clone();
let retrying_heals_for_spawn = retrying_heals_clone.clone();
let retry_task_aliases = task_aliases_clone.clone();
let retry_displaced_terminals = displaced_terminals_clone.clone();
let retry_mrf_repair_notice_targets = mrf_repair_notice_targets_clone.clone();
let retry_completed_heals = completed_heals_clone.clone();
let retry_notify = notify_clone.clone();
@@ -430,6 +436,14 @@ impl HealManager {
let admission = admission_decision.result;
let should_notify = matches!(admission, HealAdmissionResult::Accepted)
&& retry_config.event_driven_scheduler_enable;
// Publish the terminal synchronously while the
// queue transition is protected. The subsequent
// queue -> retrying handoff retains the lock order
// used by operations_snapshot.
let displaced_terminal = admission_decision
.displaced_request
.as_ref()
.map(|request| record_displaced_terminal(&retry_displaced_terminals, request));
match admission {
HealAdmissionResult::Accepted => {
// Transfer ownership while holding queue -> retrying,
@@ -437,10 +451,18 @@ impl HealManager {
#[cfg(test)]
pause_retry_ownership_transition(&retry_request_id, true).await;
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
let displaced_task_id = admission_decision.displaced_task_id;
let displaced_task_id = admission_decision.displaced_task_id().map(ToOwned::to_owned);
drop(queue);
if let Some(displaced_task_id) = displaced_task_id {
remove_task_aliases_for_task(&retry_task_aliases, &displaced_task_id).await;
if let (Some(displaced_task_id), Some(displaced_terminal)) =
(displaced_task_id, displaced_terminal)
{
remove_displaced_task_aliases(
&retry_task_aliases,
&retry_displaced_terminals,
&displaced_task_id,
&displaced_terminal,
)
.await;
remove_mrf_repair_notice_targets(
&retry_mrf_repair_notice_targets,
&displaced_task_id,
crates/heal/src/heal/manager/tests.rs
+262 -1
View File
@@ -84,6 +84,7 @@ async fn process_manager_queue_once(manager: &HealManager) {
heal_queue: &manager.heal_queue,
active_heals: &manager.active_heals,
completed_heals: &manager.completed_heals,
displaced_terminals: &manager.displaced_terminals,
task_aliases: &manager.task_aliases,
retrying_heals: &manager.retrying_heals,
mrf_repair_notice_targets: &manager.mrf_repair_notice_targets,
@@ -2778,7 +2779,10 @@ async fn test_high_priority_request_displaces_lower_priority_when_queue_full() {
HealAdmissionResult::Accepted
);
assert_eq!(manager.get_queue_length().await, 1);
assert!(matches!(manager.get_task_status(&low_id).await, Err(Error::TaskNotFound { .. })));
assert!(matches!(
manager.get_task_status(&low_id).await,
Ok(HealTaskStatus::Failed { error }) if error.contains("reason=displaced")
));
assert_eq!(
manager
.get_task_status(&high_id)
@@ -2788,6 +2792,263 @@ async fn test_high_priority_request_displaces_lower_priority_when_queue_full() {
);
}
#[tokio::test]
async fn displaced_task_remains_queryable() {
let manager = HealManager::new(
Arc::new(MockStorage),
Some(HealConfig {
queue_size: 1,
..HealConfig::default()
}),
);
let mut displaced = HealRequest::new(
HealType::Bucket {
bucket: "displaced-bucket".to_string(),
},
HealOptions::default(),
HealPriority::Low,
);
displaced.id = "displaced-task".to_string();
let displaced_id = displaced.id.clone();
manager
.submit_heal_request(displaced)
.await
.expect("displaced request should queue");
let successor = HealRequest::new(
HealType::Bucket {
bucket: "successor-bucket".to_string(),
},
HealOptions::default(),
HealPriority::High,
);
manager
.submit_heal_request(successor)
.await
.expect("successor should displace low work");
let report = manager
.get_task_report(&displaced_id)
.await
.expect("displaced report should remain queryable");
assert!(matches!(report.status, HealTaskStatus::Failed { ref error } if error.contains("reason=displaced")));
}
#[tokio::test]
async fn displaced_archive_failure_keeps_queryable_terminal() {
let manager = HealManager::new(Arc::new(MockStorage), None);
let mut request = HealRequest::new(
HealType::Bucket {
bucket: "archive-failure".to_string(),
},
HealOptions::default(),
HealPriority::Low,
);
request.id = "archive-failure-task".to_string();
let request_id = request.id.clone();
// The synchronous sidecar is the authoritative fallback when the normal
// completed-task archive has no entry (the failure window that must not
// turn an Accepted ID into NotFound).
record_displaced_terminal(&manager.displaced_terminals, &request);
assert!(manager.completed_heals.lock().await.is_empty());
assert!(matches!(
manager.get_task_status(&request_id).await,
Ok(HealTaskStatus::Failed { error }) if error.contains("reason=displaced")
));
}
#[tokio::test]
async fn scheduler_retry_displacement_keeps_evicted_task_queryable() {
let manager = Arc::new(HealManager::new(
Arc::new(MockStorage),
Some(HealConfig {
queue_size: 1,
event_driven_scheduler_enable: false,
..HealConfig::default()
}),
));
let mut retry_request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
retry_request.priority = HealPriority::High;
let retry_id = retry_request.id.clone();
manager
.submit_heal_request(retry_request)
.await
.expect("retry request should queue");
// Process exactly one queue cycle so the retry task is spawned without a
// background scheduler consuming the filler request before the retry wakes.
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if manager.retrying_heals.lock().await.contains_key(&retry_id) {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("retry request should enter backoff");
let filler = HealRequest::new(
HealType::Bucket {
bucket: "retry-displaced-filler".to_string(),
},
HealOptions::default(),
HealPriority::Low,
);
let filler_id = filler.id.clone();
manager
.submit_heal_request(filler)
.await
.expect("filler request should occupy the queue");
tokio::time::timeout(Duration::from_secs(5), async {
loop {
if matches!(
manager.get_task_status(&filler_id).await,
Ok(HealTaskStatus::Failed { ref error }) if error.contains("reason=displaced")
) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("retry admission should displace the filler request");
assert_eq!(manager.get_queue_length().await, 1);
assert_eq!(
manager.get_task_status(&retry_id).await.expect("retry should be queued"),
HealTaskStatus::Pending
);
}
#[tokio::test]
async fn concurrent_displacers_produce_one_terminal_generation() {
let manager = Arc::new(HealManager::new(
Arc::new(MockStorage),
Some(HealConfig {
queue_size: 1,
..HealConfig::default()
}),
));
let mut displaced = HealRequest::new(
HealType::Bucket {
bucket: "concurrent-displaced".to_string(),
},
HealOptions::default(),
HealPriority::Low,
);
displaced.id = "concurrent-displaced-task".to_string();
let displaced_id = displaced.id.clone();
manager
.submit_heal_request(displaced)
.await
.expect("initial request should queue");
let first = HealRequest::new(
HealType::Bucket {
bucket: "concurrent-successor-a".to_string(),
},
HealOptions::default(),
HealPriority::High,
);
let second = HealRequest::new(
HealType::Bucket {
bucket: "concurrent-successor-b".to_string(),
},
HealOptions::default(),
HealPriority::High,
);
let (first_result, second_result) = tokio::join!(manager.submit_heal_request(first), manager.submit_heal_request(second));
let accepted = [&first_result, &second_result]
.into_iter()
.filter(|result| matches!(result, Ok(HealAdmissionResult::Accepted)))
.count();
assert_eq!(accepted, 1, "exactly one concurrent displacer should win the full queue");
assert!(
first_result.is_ok() && second_result.is_ok(),
"the losing request should receive a typed Full result"
);
let terminals = lock_displaced_terminals(&manager.displaced_terminals);
assert_eq!(terminals.len(), 1);
assert!(terminals.contains_key(&displaced_id));
}
#[tokio::test]
async fn successor_chain_is_bounded_and_authorized() {
let manager = HealManager::new(
Arc::new(MockStorage),
Some(HealConfig {
queue_size: 1,
..HealConfig::default()
}),
);
let mut original = HealRequest::new(
HealType::Bucket {
bucket: "authorized-original".to_string(),
},
HealOptions::default(),
HealPriority::Low,
);
original.id = "authorized-original-task".to_string();
let original_id = original.id.clone();
manager.submit_heal_request(original).await.expect("original should queue");
let mut duplicate = HealRequest::new(
HealType::Bucket {
bucket: "authorized-original".to_string(),
},
HealOptions::default(),
HealPriority::Low,
);
duplicate.id = "authorized-duplicate-task".to_string();
let duplicate_id = duplicate.id.clone();
manager
.submit_heal_request(duplicate)
.await
.expect("same-target duplicate should merge");
let successor = HealRequest::new(
HealType::Bucket {
bucket: "authorized-successor".to_string(),
},
HealOptions::default(),
HealPriority::High,
);
let successor_id = successor.id.clone();
manager.submit_heal_request(successor).await.expect("successor should queue");
assert!(manager.task_aliases.lock().await.is_empty());
assert!(matches!(manager.get_task_status(&original_id).await, Ok(HealTaskStatus::Failed { .. })));
assert!(matches!(manager.get_task_status(&duplicate_id).await, Ok(HealTaskStatus::Failed { .. })));
assert_eq!(
manager
.get_task_status(&successor_id)
.await
.expect("successor should remain queued"),
HealTaskStatus::Pending
);
}
#[tokio::test]
async fn displaced_terminal_expires_after_bounded_ttl() {
let manager = HealManager::new(Arc::new(MockStorage), None);
let mut request = HealRequest::new(
HealType::Bucket {
bucket: "expires".to_string(),
},
HealOptions::default(),
HealPriority::Low,
);
request.id = "expires-task".to_string();
let request_id = request.id.clone();
record_displaced_terminal(&manager.displaced_terminals, &request);
{
let mut terminals = lock_displaced_terminals(&manager.displaced_terminals);
let entry =
Arc::get_mut(terminals.get_mut(&request_id).expect("terminal should be retained")).expect("test owns terminal entry");
entry.completed_at = SystemTime::now() - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_secs(1);
}
assert!(matches!(manager.get_task_status(&request_id).await, Err(Error::TaskNotFound { .. })));
}
#[tokio::test]
async fn test_displacing_registered_mrf_task_drops_notice_ownership() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
crates/heal/src/heal/task/heal_erasure_set.rs
+9 -2
View File
@@ -231,6 +231,10 @@ impl HealTask {
"Heal erasure set format repair skipped because no format heal was required"
);
} else {
let error = e;
if error.is_recoverable_heal() {
return Err(error);
}
error!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT,
@@ -239,7 +243,7 @@ impl HealTask {
task_id = %self.id,
set_disk_id,
result = "format_failed",
error = %e,
error = %error,
"Heal erasure set failed"
);
{
@@ -247,7 +251,7 @@ impl HealTask {
progress.update_progress(4, 4, 0, 0);
}
return Err(Error::TaskExecutionFailed {
message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
message: format!("Failed to heal disk format for {set_disk_id}: {error}"),
});
}
} else {
@@ -284,6 +288,9 @@ impl HealTask {
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
Err(e) => {
if e.is_recoverable_heal() {
return Err(e);
}
error!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT,
crates/heal/src/heal/task/tests.rs
+28
View File
@@ -547,6 +547,7 @@ struct MockStorage {
heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
format_no_heal_required: Mutex<bool>,
format_error: Mutex<Option<Error>>,
global_format_calls: Mutex<u32>,
replacement_format_calls: Mutex<Vec<(usize, usize, Vec<String>)>>,
replacement_targets_ready: Mutex<bool>,
@@ -867,6 +868,9 @@ impl HealStorageAPI for MockStorage {
async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
*self.global_format_calls.lock().unwrap() += 1;
if let Some(error) = self.format_error.lock().unwrap().take() {
return Err(error);
}
let no_heal_required = *self.format_no_heal_required.lock().unwrap();
if no_heal_required {
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::NoHealRequired))))
@@ -2052,6 +2056,30 @@ async fn test_erasure_set_heal_continues_after_format_no_heal_required() {
);
}
#[tokio::test]
async fn erasure_set_format_slowdown_is_propagated() {
let storage = Arc::new(MockStorage {
format_error: Mutex::new(Some(Error::Storage(EcstoreError::SlowDown))),
..Default::default()
});
let request = HealRequest::new(
HealType::ErasureSet {
buckets: Vec::new(),
set_disk_id: "pool_0_set_0".to_string(),
},
HealOptions::default(),
HealPriority::Normal,
);
let task = HealTask::from_request(request, storage);
let error = task
.execute()
.await
.expect_err("format SlowDown must remain recoverable for the task manager");
assert!(matches!(error, Error::Storage(EcstoreError::SlowDown)));
}
#[tokio::test]
async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
let temp = TempDir::new().expect("temporary directory should be created");
crates/iam/AGENTS.md
-1
View File
@@ -24,4 +24,3 @@ Applies to `crates/iam/`.
## Suggested Validation
- `cargo test -p rustfs-iam`
- Full gate before commit: `make pre-commit`
crates/io-metrics/src/internode_metrics.rs
+7
View File
@@ -54,6 +54,13 @@ pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_versio
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode";
pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip";
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE: &str = "batch_read_version_request_encode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE: &str = "batch_read_version_request_decode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ: &str = "batch_read_version_disk_read";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch_read_version_response_json_encode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode";
const OPERATION_LABEL: &str = "operation";
const BACKEND_LABEL: &str = "backend";
crates/lifecycle/Cargo.toml
+1 -1
View File
@@ -69,7 +69,7 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnos
[dev-dependencies]
metrics-util = { workspace = true, features = ["debugging"] }
proptest = "1"
serial_test.workspace = true
serial_test = { workspace = true }
temp-env.workspace = true
tokio = { workspace = true, features = ["macros", "fs", "rt-multi-thread"] }
crates/lifecycle/src/core.rs
-45
View File
@@ -1564,7 +1564,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn abort_incomplete_multipart_upload_due_accepts_zero_days() {
let initiated = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -1625,7 +1624,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn predict_expiration_selects_closest_expiry_for_put_object() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -1872,7 +1870,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn empty_transition_vectors_are_not_active_or_due() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1938,7 +1935,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_keeps_latest_object_before_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -1972,7 +1968,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_transitions_latest_object_after_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2010,7 +2005,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_transitions_latest_object_after_date_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let transition_date = base_time - Duration::days(1);
@@ -2050,7 +2044,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_selects_earliest_due_among_multiple_past_due_events() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
// Two enabled rules both yield a past-due DeleteAction and a third yields a
@@ -2164,7 +2157,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_expires_noncurrent_version_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2202,7 +2194,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_skips_noncurrent_expiration_without_successor() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
let lc = BucketLifecycleConfiguration {
@@ -2238,7 +2229,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_missing_successor_does_not_skip_noncurrent_transition() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp");
let lc = BucketLifecycleConfiguration {
@@ -2281,7 +2271,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_noncurrent_expiration_one_day_respects_due_boundary() {
let successor_time = datetime!(2025-06-15 12:00:00 UTC);
let due = expected_expiry_time(successor_time, 1);
@@ -2323,7 +2312,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_expires_noncurrent_version_immediately_when_zero_days() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2361,7 +2349,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_transitions_noncurrent_version_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2437,7 +2424,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn evaluator_honors_newer_noncurrent_versions_retention_count() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = Arc::new(BucketLifecycleConfiguration {
@@ -2726,7 +2712,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expired_object_delete_marker_ignores_marker_with_noncurrent_versions_present() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2803,7 +2788,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expired_object_delete_marker_deletes_only_delete_marker_immediately() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2881,7 +2865,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expiration_days_deletes_only_expired_delete_marker_when_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -2932,7 +2915,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expiration_days_uses_earliest_due_rule_for_expired_delete_marker() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let make_rule = |id: &str, days| LifecycleRule {
@@ -3263,7 +3245,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn del_marker_expiration_deletes_marker_and_older_versions_when_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
let lc = BucketLifecycleConfiguration {
@@ -3303,7 +3284,6 @@ mod tests {
// --- TASK-003 tests: Round up to next UTC processing boundary ---
#[test]
#[serial]
fn expected_expiry_time_rounds_up_to_next_midnight_utc() {
with_default_ilm_process_time(|| {
// Object created at 2025-01-15T10:30:45Z, expire in 30 days
@@ -3319,7 +3299,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_immediate_expiry_returns_epoch() {
with_default_ilm_process_time(|| {
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
@@ -3329,7 +3308,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_preserves_exact_midnight_boundary() {
with_default_ilm_process_time(|| {
let mod_time = datetime!(2025-03-01 00:00:00 UTC);
@@ -3339,7 +3317,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_rounds_end_of_day_to_following_midnight() {
with_default_ilm_process_time(|| {
let mod_time = datetime!(2025-06-15 23:59:59 UTC);
@@ -3349,7 +3326,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_uses_canonical_process_time_boundary() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
@@ -3362,7 +3338,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_uses_deprecated_process_time_alias() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
@@ -3375,7 +3350,6 @@ mod tests {
}
#[test]
#[serial]
fn expected_expiry_time_uses_default_boundary_when_process_time_is_zero_or_invalid() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
@@ -3398,7 +3372,6 @@ mod tests {
// (a) Default path (env unset) is byte-identical: one day == 86400s.
#[test]
#[serial]
fn ilm_day_secs_defaults_to_86400_when_unset() {
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
@@ -3427,7 +3400,6 @@ mod tests {
// (b) End-to-end env read scales the day length.
#[test]
#[serial]
fn ilm_day_secs_scales_when_env_set() {
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
assert_eq!(ilm_day_secs(), 2);
@@ -3436,7 +3408,6 @@ mod tests {
// (c) Invalid env value falls back to 86400.
#[test]
#[serial]
fn ilm_day_secs_falls_back_on_invalid_env() {
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("bogus"), || {
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
@@ -3449,7 +3420,6 @@ mod tests {
// Deadline math scales: with a 1s day and PROCESS_TIME unset, a Days=1 rule is
// due 1s after mod_time (rounded up to the next 1s boundary => same instant).
#[test]
#[serial]
fn expected_expiry_time_scales_with_debug_day_secs() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("1"), || {
@@ -3465,7 +3435,6 @@ mod tests {
// days == 0 still yields the immediate-expiry sentinel regardless of the switch.
#[test]
#[serial]
fn expected_expiry_time_zero_days_ignores_debug_day_secs() {
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
@@ -3476,7 +3445,6 @@ mod tests {
// (③) Interaction with an explicit RUSTFS_ILM_PROCESS_TIME: the deadline offset
// uses the accelerated day length, but the rounding boundary honors PROCESS_TIME.
#[test]
#[serial]
fn expected_expiry_time_debug_day_secs_respects_explicit_process_time() {
let mod_time = datetime!(2025-01-15 10:30:00 UTC);
// day == 10s, but round up to the next 60s (PROCESS_TIME) boundary.
@@ -3493,7 +3461,6 @@ mod tests {
// (③) With the switch unset, an explicit PROCESS_TIME behaves exactly as before.
#[test]
#[serial]
fn expected_expiry_time_unset_debug_day_secs_matches_legacy_process_time() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
@@ -3521,7 +3488,6 @@ mod tests {
// The abort-incomplete-multipart deadline path also scales through the switch.
#[test]
#[serial]
fn abort_incomplete_multipart_due_scales_with_debug_day_secs() {
use s3s::dto::AbortIncompleteMultipartUpload;
let initiated = datetime!(2025-01-15 10:30:45 UTC);
@@ -3566,7 +3532,6 @@ mod tests {
// (⑤ evaluator seam) A Days=1 rule fires under RUSTFS_ILM_DEBUG_DAY_SECS=1 once
// `now` advances a few seconds past a mod_time only ~seconds in the past.
#[test]
#[serial]
fn eval_inner_expires_days_one_rule_under_debug_day_secs() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3615,7 +3580,6 @@ mod tests {
// Absolute Date-based rules must NOT scale with the switch (regression guard).
#[test]
#[serial]
fn eval_inner_date_rule_ignores_debug_day_secs() {
let expiry_date = datetime!(2025-06-01 00:00:00 UTC);
let lc = BucketLifecycleConfiguration {
@@ -3873,7 +3837,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_triggers_delete_all_versions_when_expired_object_all_versions_set() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -3912,7 +3875,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn expired_object_all_versions_does_not_apply_to_current_delete_marker() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
let lc = BucketLifecycleConfiguration {
@@ -3942,7 +3904,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn eval_inner_uses_delete_action_when_all_versions_not_set() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
@@ -4061,7 +4022,6 @@ mod tests {
use super::*;
use proptest::prelude::*;
use s3s::dto::{NoncurrentVersionExpiration, Tag};
use serial_test::serial;
const DAY_SECS: i64 = 86400;
@@ -4292,7 +4252,6 @@ mod tests {
/// combination, and must be deterministic: the same input
/// evaluated twice yields an identical event.
#[test]
#[serial]
fn eval_inner_never_panics_and_is_deterministic(
rules in prop::collection::vec(arb_rule(), 0..4),
obj in arb_object_opts(),
@@ -4432,7 +4391,6 @@ mod tests {
/// candidate set — earliest due wins, ties prefer delete-class —
/// and must be `NoneAction` exactly when that set is empty.
#[test]
#[serial]
fn eval_inner_winner_matches_selection_oracle(
rules in prop::collection::vec(arb_selection_rule(), 0..5),
mod_off in 0i64..(2 * DAY_SECS),
@@ -4486,7 +4444,6 @@ mod tests {
/// non-decreasing in `days` (days == 0 maps to UNIX_EPOCH, below
/// any post-1970 deadline).
#[test]
#[serial]
fn expected_expiry_time_is_monotonic_in_days(
mod_off in 0i64..(3650 * DAY_SECS),
d1 in 0i32..2000,
@@ -4508,7 +4465,6 @@ mod tests {
/// to the next whole-day boundary: the result is day-aligned, not
/// before `mod_time + days`, and less than one boundary beyond it.
#[test]
#[serial]
fn expected_expiry_time_lands_on_default_day_boundary(
mod_off in 0i64..(3650 * DAY_SECS),
days in 1i32..2000,
@@ -4526,7 +4482,6 @@ mod tests {
/// to that boundary instead: aligned to it, never early, and less
/// than one boundary late.
#[test]
#[serial]
fn expected_expiry_time_lands_on_explicit_process_boundary(
mod_off in 0i64..(365 * DAY_SECS),
days in 1i32..400,
crates/madmin/src/metrics.rs
+16
View File
@@ -689,6 +689,14 @@ pub struct ScannerMetrics {
pub cycle_max_objects: u64,
#[serde(rename = "cycle_max_directories", default)]
pub cycle_max_directories: u64,
#[serde(rename = "cycle_timeout_total", default)]
pub cycle_timeout_total: u64,
#[serde(rename = "cycle_recovery_required_total", default)]
pub cycle_recovery_required_total: u64,
#[serde(rename = "cycle_last_progress_age", default)]
pub cycle_last_progress_age: u64,
#[serde(rename = "leader_lease_without_progress", default)]
pub leader_lease_without_progress: bool,
#[serde(rename = "bitrot_cycle_enabled", default)]
pub bitrot_cycle_enabled: bool,
#[serde(rename = "bitrot_cycle_seconds", default)]
@@ -764,6 +772,8 @@ impl ScannerMetrics {
self.cycle_max_duration_seconds = other.cycle_max_duration_seconds;
self.cycle_max_objects = other.cycle_max_objects;
self.cycle_max_directories = other.cycle_max_directories;
self.cycle_last_progress_age = other.cycle_last_progress_age;
self.leader_lease_without_progress = other.leader_lease_without_progress;
self.bitrot_cycle_enabled = other.bitrot_cycle_enabled;
self.bitrot_cycle_seconds = other.bitrot_cycle_seconds;
}
@@ -857,6 +867,12 @@ impl ScannerMetrics {
.saturating_add(other.last_cycle_replication_checks);
self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves);
self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles);
self.cycle_timeout_total = self.cycle_timeout_total.saturating_add(other.cycle_timeout_total);
self.cycle_recovery_required_total = self
.cycle_recovery_required_total
.saturating_add(other.cycle_recovery_required_total);
self.cycle_last_progress_age = self.cycle_last_progress_age.max(other.cycle_last_progress_age);
self.leader_lease_without_progress |= other.leader_lease_without_progress;
self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles);
self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown);
self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime);
crates/madmin/src/trace.rs
+32 -138
View File
@@ -12,18 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{collections::HashMap, time::Duration};
use jiff::Timestamp;
use serde::{Deserialize, Serialize};
use crate::heal_commands::HealResultItem;
/// Bitflag helper for service trace categories.
///
/// Each variant occupies a single bit so that a `TraceType` value can represent
/// an arbitrary combination of categories via bitwise OR.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
pub struct TraceType(u64);
impl TraceType {
// Define some constants
pub const OS: TraceType = TraceType(1 << 0);
pub const STORAGE: TraceType = TraceType(1 << 1);
pub const S3: TraceType = TraceType(1 << 2);
@@ -40,15 +38,13 @@ impl TraceType {
pub const FTP: TraceType = TraceType(1 << 13);
pub const ILM: TraceType = TraceType(1 << 14);
// MetricsAll must be last.
/// All trace categories combined. Must be updated when adding new variants.
pub const ALL: TraceType = TraceType((1 << 15) - 1);
pub fn new(t: u64) -> Self {
Self(t)
}
}
impl TraceType {
pub fn contains(&self, x: &TraceType) -> bool {
(self.0 & x.0) == x.0
}
@@ -76,140 +72,38 @@ impl TraceType {
}
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct TraceInfo {
#[serde(rename = "type")]
trace_type: u64,
#[serde(rename = "nodename")]
node_name: String,
#[serde(rename = "funcname")]
func_name: String,
#[serde(rename = "time")]
time: Timestamp,
#[serde(rename = "path")]
path: String,
#[serde(rename = "dur")]
duration: Duration,
#[serde(rename = "bytes", skip_serializing_if = "Option::is_none")]
bytes: Option<i64>,
#[serde(rename = "msg", skip_serializing_if = "Option::is_none")]
message: Option<String>,
#[serde(rename = "error", skip_serializing_if = "Option::is_none")]
error: Option<String>,
#[serde(rename = "custom", skip_serializing_if = "Option::is_none")]
custom: Option<HashMap<String, String>>,
#[serde(rename = "http", skip_serializing_if = "Option::is_none")]
http: Option<TraceHTTPStats>,
#[serde(rename = "healResult", skip_serializing_if = "Option::is_none")]
heal_result: Option<HealResultItem>,
}
impl TraceInfo {
pub fn mask(&self) -> u64 {
TraceType::new(self.trace_type).mask()
}
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct TraceInfoLegacy {
trace_info: TraceInfo,
#[serde(rename = "request")]
req_info: Option<TraceRequestInfo>,
#[serde(rename = "response")]
resp_info: Option<TraceResponseInfo>,
#[serde(rename = "stats")]
call_stats: Option<TraceCallStats>,
#[serde(rename = "storageStats")]
storage_stats: Option<StorageStats>,
#[serde(rename = "osStats")]
os_stats: Option<OSStats>,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct StorageStats {
path: String,
duration: Duration,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct OSStats {
path: String,
duration: Duration,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct TraceHTTPStats {
req_info: TraceRequestInfo,
resp_info: TraceResponseInfo,
call_stats: TraceCallStats,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct TraceCallStats {
input_bytes: i32,
output_bytes: i32,
latency: Duration,
time_to_first_byte: Duration,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct TraceRequestInfo {
time: Timestamp,
proto: String,
method: String,
#[serde(skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
raw_query: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
headers: Option<HashMap<String, String>>,
#[serde(skip_serializing_if = "Option::is_none")]
body: Option<Vec<u8>>,
client: String,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct TraceResponseInfo {
time: Timestamp,
#[serde(skip_serializing_if = "Option::is_none")]
headers: Option<HashMap<String, String>>,
#[serde(skip_serializing_if = "Option::is_none")]
body: Option<Vec<u8>>,
#[serde(skip_serializing_if = "Option::is_none")]
status_code: Option<i32>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn trace_timestamps_serialize_as_rfc3339_utc() {
let timestamp = Timestamp::constant(1_700_000_000, 123_456_000);
let trace = TraceInfo {
time: timestamp,
http: Some(TraceHTTPStats {
req_info: TraceRequestInfo {
time: timestamp,
..Default::default()
},
resp_info: TraceResponseInfo {
time: timestamp,
..Default::default()
},
..Default::default()
}),
..Default::default()
};
fn trace_type_contains_and_overlaps() {
let mut combined = TraceType::default();
combined.merge(&TraceType::S3);
combined.merge(&TraceType::HEALING);
let value = serde_json::to_value(trace).expect("trace should serialize");
assert_eq!(value["time"], "2023-11-14T22:13:20.123456Z");
assert_eq!(value["http"]["req_info"]["time"], "2023-11-14T22:13:20.123456Z");
assert_eq!(value["http"]["resp_info"]["time"], "2023-11-14T22:13:20.123456Z");
let trace: TraceInfo = serde_json::from_value(value).expect("trace should deserialize");
assert_eq!(trace.time, timestamp);
let http = trace.http.expect("http trace should deserialize");
assert_eq!(http.req_info.time, timestamp);
assert_eq!(http.resp_info.time, timestamp);
assert!(combined.contains(&TraceType::S3));
assert!(combined.contains(&TraceType::HEALING));
assert!(!combined.contains(&TraceType::SCANNER));
assert!(combined.overlaps(&TraceType::S3));
assert!(combined.overlaps(&TraceType::HEALING));
assert!(!combined.overlaps(&TraceType::SCANNER));
}
#[test]
fn trace_type_set_if() {
let mut tt = TraceType::default();
tt.set_if(true, &TraceType::OS);
tt.set_if(false, &TraceType::S3);
assert!(tt.contains(&TraceType::OS));
assert!(!tt.contains(&TraceType::S3));
}
#[test]
fn trace_type_single_type() {
assert!(TraceType::S3.single_type());
let mut combined = TraceType::S3;
combined.merge(&TraceType::HEALING);
assert!(!combined.single_type());
}
}
crates/notify/AGENTS.md
-1
View File
@@ -55,4 +55,3 @@ shared plugin/runtime primitives from `rustfs-targets`.
- Focused: `cargo test -p rustfs-notify runtime_facade`
- Focused: `cargo test -p rustfs-notify runtime_view`
- Focused: `cargo test -p rustfs-notify config_manager`
- Full gate before commit: `make pre-commit`
crates/object-capacity/Cargo.toml
-1
View File
@@ -73,7 +73,6 @@ walkdir = { workspace = true }
[dev-dependencies]
criterion = { workspace = true, features = ["html_reports"] }
serial_test = { workspace = true }
temp-env = { workspace = true, features = ["async_closure"] }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
crates/object-capacity/src/capacity_manager.rs
-34
View File
@@ -1484,7 +1484,6 @@ mod tests {
ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
ENV_CAPACITY_WRITE_TRIGGER_DELAY,
};
use serial_test::serial;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -1669,7 +1668,6 @@ mod tests {
}
#[test]
#[serial]
fn test_config_getter_defaults() {
for (env_var, getter, default, _, _) in config_getter_cases() {
temp_env::with_var(env_var, None::<&str>, || {
@@ -1679,7 +1677,6 @@ mod tests {
}
#[test]
#[serial]
fn test_config_getter_env_overrides() {
for (env_var, getter, _, override_value, expected) in config_getter_cases() {
temp_env::with_var(env_var, Some(override_value), || {
@@ -1689,7 +1686,6 @@ mod tests {
}
#[test]
#[serial]
fn test_zero_env_values_clamp_to_defaults() {
// A zero threshold makes small disks report 0 bytes; a zero timeout
// (with dynamic timeout off) makes every scan fail. Both must fall
@@ -1709,7 +1705,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_preserves_retrieval_metadata() {
let manager = HybridCapacityManager::from_env();
@@ -1725,7 +1720,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_record_write_operation() {
let manager = HybridCapacityManager::from_env();
@@ -1736,7 +1730,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_write_frequency_window() {
let manager = HybridCapacityManager::from_env();
@@ -1824,7 +1817,6 @@ mod tests {
}
#[test]
#[serial]
fn test_recent_write_count_ignores_future_buckets() {
let record = WriteRecord::new();
record.write_buckets[0].store(120, 3);
@@ -1838,7 +1830,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_needs_fast_update() {
let manager = HybridCapacityManager::from_env();
@@ -1855,7 +1846,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_cache_age_tracking() {
let manager = HybridCapacityManager::from_env();
@@ -1875,7 +1865,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_data_source_tracking() {
let manager = HybridCapacityManager::from_env();
@@ -1891,7 +1880,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_needs_fast_update_waits_for_write_trigger_delay() {
let manager = create_isolated_manager(HybridStrategyConfig {
scheduled_update_interval: Duration::from_secs(60),
@@ -1922,7 +1910,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_needs_fast_update_respects_enable_write_trigger() {
let manager = create_isolated_manager(HybridStrategyConfig {
scheduled_update_interval: Duration::from_secs(60),
@@ -1949,7 +1936,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_concurrent_access() {
let manager = Arc::new(HybridCapacityManager::from_env());
let mut handles = Vec::new();
@@ -1976,7 +1962,6 @@ mod tests {
// exact under heavy same-second contention or the frequency window (and the
// write-trigger decision) would undercount.
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
#[serial]
async fn test_record_write_operation_lock_free_is_exact_under_contention() {
let manager = Arc::new(HybridCapacityManager::from_env());
let mut handles = Vec::new();
@@ -2001,7 +1986,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_performance_overhead() {
let manager = Arc::new(HybridCapacityManager::from_env());
let start = Instant::now();
@@ -2018,7 +2002,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_refresh_or_join_singleflight() {
let manager = Arc::new(HybridCapacityManager::from_env());
let calls = Arc::new(AtomicUsize::new(0));
@@ -2058,7 +2041,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_refresh_or_join_recovers_after_leader_cancellation() {
let manager = Arc::new(HybridCapacityManager::from_env());
@@ -2087,7 +2069,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_refresh_or_join_cancelled_leader_unblocks_joiner() {
let manager = Arc::new(HybridCapacityManager::from_env());
@@ -2115,7 +2096,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_spawn_refresh_if_needed_deduplicates_background_refresh() {
let manager = Arc::new(HybridCapacityManager::from_env());
let calls = Arc::new(AtomicUsize::new(0));
@@ -2153,7 +2133,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_record_write_operation_with_scope_token_marks_dirty_disks() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
let token = uuid::Uuid::new_v4();
@@ -2177,7 +2156,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_get_dirty_disks_drains_global_dirty_scope_registry() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
record_global_dirty_scope(CapacityScope {
@@ -2197,7 +2175,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_recomputes_total_from_disk_cache_for_subset_refresh() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2308,7 +2285,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_degraded_full_refresh_merges_cache_and_does_not_oscillate() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2354,7 +2330,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_degraded_with_empty_per_disk_serves_merged_cache() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
manager.update_capacity(full_two_disk_update(), DataSource::RealTime).await;
@@ -2384,7 +2359,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_update_capacity_degraded_without_complete_cache_keeps_partial_sum() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2425,7 +2399,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_commit_keeps_dirty_marks_recorded_after_scan_start() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
let disk = scope_disk("node-a", "/tmp/disk-a");
@@ -2456,7 +2429,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_commit_clears_dirty_marks_recorded_before_scan_start() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
let disk = scope_disk("node-a", "/tmp/disk-a");
@@ -2477,7 +2449,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_retain_dirty_disks_within_drops_ghost_entries() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
let local = scope_disk("node-a", "/tmp/disk-a");
@@ -2496,7 +2467,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_spawn_refresh_recovers_from_construction_panic() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2563,7 +2533,6 @@ mod tests {
}
#[tokio::test(start_paused = true)]
#[serial]
async fn test_refresh_or_join_joiner_times_out_when_leader_wedges() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2591,7 +2560,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_refresh_or_join_returns_cluster_total_for_dirty_subset() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
@@ -2673,7 +2641,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_config_from_env() {
let config = HybridStrategyConfig::from_env();
@@ -2687,7 +2654,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_config_from_env_with_override() {
temp_env::with_var(ENV_CAPACITY_SCHEDULED_INTERVAL, Some("600"), || {
let config = HybridStrategyConfig::from_env();
crates/object-capacity/src/scan.rs
-5
View File
@@ -1069,7 +1069,6 @@ mod tests {
#[cfg(unix)]
use rustfs_config::ENV_CAPACITY_FOLLOW_SYMLINKS;
use rustfs_config::{ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_SAMPLE_RATE};
use serial_test::serial;
/// Reference implementation using unbounded `u128` arithmetic, clamped to
/// `u64::MAX`, used as the source of truth for the sampling extrapolation.
@@ -1274,7 +1273,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_get_dir_size_async_nonexistent_directory() {
let result = get_dir_size_async(Path::new("/nonexistent/path")).await;
assert!(result.is_err());
@@ -1648,7 +1646,6 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn test_metadata_incomplete_aggregate_does_not_replace_disk_cache() {
use std::fs::File;
use std::io::Write;
@@ -1783,7 +1780,6 @@ mod tests {
#[cfg(unix)]
#[tokio::test]
#[serial]
async fn test_get_dir_size_async_ignores_symlink_targets_when_follow_disabled() {
use std::fs::File;
use std::io::Write;
@@ -1809,7 +1805,6 @@ mod tests {
#[cfg(unix)]
#[tokio::test]
#[serial]
async fn test_get_dir_size_async_counts_symlink_targets_when_follow_enabled() {
use std::fs::File;
use std::io::Write;
crates/obs/src/metrics/collectors/bucket_replication.rs
+5 -5
View File
@@ -75,7 +75,7 @@ pub struct BucketReplicationBandwidthStats {
}
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationStats {
pub struct BucketReplicationMetricsSnapshot {
pub bucket: String,
pub total_failed_bytes: u64,
pub total_failed_count: u64,
@@ -107,7 +107,7 @@ pub struct BucketReplicationStats {
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationRuntimeStats {
pub(crate) stats: BucketReplicationStats,
pub(crate) stats: BucketReplicationMetricsSnapshot,
pub(crate) target_flows: Vec<BucketReplicationTargetFlowStats>,
}
@@ -182,7 +182,7 @@ fn push_proxy_request_result_metrics(
}
}
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationMetricsSnapshot]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
@@ -572,7 +572,7 @@ mod tests {
#[test]
fn test_collect_bucket_replication_metrics() {
let stats = vec![BucketReplicationRuntimeStats {
stats: BucketReplicationStats {
stats: BucketReplicationMetricsSnapshot {
bucket: "b1".to_string(),
total_failed_bytes: 64,
total_failed_count: 2,
@@ -876,7 +876,7 @@ mod tests {
#[test]
fn test_collect_bucket_replication_metrics_empty() {
let stats: Vec<BucketReplicationStats> = Vec::new();
let stats: Vec<BucketReplicationMetricsSnapshot> = Vec::new();
let metrics = collect_bucket_replication_metrics(&stats);
assert!(metrics.is_empty());
}
crates/obs/src/metrics/collectors/mod.rs
+2 -2
View File
@@ -48,7 +48,7 @@ pub(crate) use bucket_replication::{
BucketReplicationTargetFlowStats, collect_bucket_replication_backlog_metrics, collect_bucket_replication_runtime_metrics,
};
pub use bucket_replication::{
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
BucketReplicationBandwidthStats, BucketReplicationMetricsSnapshot, BucketReplicationTargetStats,
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
};
pub use cluster::{ClusterStats, collect_cluster_metrics};
@@ -68,8 +68,8 @@ pub(crate) use notification::collect_notification_runtime_metrics;
pub use notification::{NotificationStats, collect_notification_metrics};
pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
pub use replication::{ReplicationMetricsSnapshot, collect_replication_metrics};
pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics};
pub use replication::{ReplicationStats, collect_replication_metrics};
pub(crate) use request::{ApiRequestMetricSupport, ApiRequestStats, collect_request_metrics};
pub use resource::{ResourceStats, collect_resource_metrics};
pub(crate) use scanner::{ScannerRuntimeStats, collect_scanner_runtime_metrics};
crates/obs/src/metrics/collectors/replication.rs
+6 -6
View File
@@ -22,7 +22,7 @@ use crate::metrics::schema::replication::*;
/// Replication statistics.
#[derive(Debug, Clone, Default)]
pub struct ReplicationStats {
pub struct ReplicationMetricsSnapshot {
/// Average number of active replication workers
pub average_active_workers: f64,
/// Average queued bytes since server start
@@ -54,13 +54,13 @@ pub struct ReplicationStats {
#[derive(Debug, Clone, Default)]
pub(crate) struct ReplicationRuntimeStats {
pub(crate) server: String,
pub(crate) stats: ReplicationStats,
pub(crate) stats: ReplicationMetricsSnapshot,
}
/// Collects replication metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for replication statistics.
pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
pub fn collect_replication_metrics(stats: &ReplicationMetricsSnapshot) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_MD, stats.average_active_workers),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_MD, stats.average_queued_bytes as f64),
@@ -120,7 +120,7 @@ mod tests {
#[test]
fn test_collect_replication_metrics() {
let stats = ReplicationStats {
let stats = ReplicationMetricsSnapshot {
average_active_workers: 8.5,
average_queued_bytes: 1024 * 1024 * 40,
average_queued_count: 240,
@@ -182,7 +182,7 @@ mod tests {
#[test]
fn test_collect_replication_metrics_default() {
let stats = ReplicationStats::default();
let stats = ReplicationMetricsSnapshot::default();
let metrics = collect_replication_metrics(&stats);
assert_eq!(metrics.len(), 13);
@@ -194,7 +194,7 @@ mod tests {
#[test]
fn replication_stats_struct_literal_keeps_legacy_fields() {
let stats = ReplicationStats {
let stats = ReplicationMetricsSnapshot {
average_active_workers: 1.0,
average_queued_bytes: 2,
average_queued_count: 3,
crates/obs/src/metrics/scheduler.rs
+2 -2
View File
@@ -2811,14 +2811,14 @@ mod tests {
#[test]
fn replication_proxy_bucket_keys_detect_removed_buckets() {
let previous = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
stats: crate::metrics::BucketReplicationStats {
stats: crate::metrics::BucketReplicationMetricsSnapshot {
bucket: "photos".to_string(),
..Default::default()
},
..Default::default()
}]);
let current = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {
stats: crate::metrics::BucketReplicationStats {
stats: crate::metrics::BucketReplicationMetricsSnapshot {
bucket: "logs".to_string(),
..Default::default()
},
crates/obs/src/metrics/stats_collector.rs
+11 -11
View File
@@ -21,12 +21,12 @@
use crate::metrics::collectors::scanner::{ScannerActiveBucketDriveStats, ScannerBucketDriveResultStats, ScannerSourceWorkStats};
use crate::metrics::collectors::{
ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats,
BucketReplicationRuntimeStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, BucketReplicationTargetFlowStats,
BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats,
DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmBackpressureStats,
IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType,
ReplicationStats, ResourceStats, ScannerRuntimeStats, ScannerStats,
BucketReplicationMetricsSnapshot, BucketReplicationRuntimeStats, BucketReplicationTargetBacklogStats,
BucketReplicationTargetFlowStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats,
ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats,
DriveDetailedStats, DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats,
IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats,
ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot, ResourceStats, ScannerRuntimeStats, ScannerStats,
};
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
use crate::metrics::{
@@ -266,7 +266,7 @@ fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnaps
BucketReplicationRuntimeStats {
target_flows,
stats: BucketReplicationStats {
stats: BucketReplicationMetricsSnapshot {
bucket,
total_failed_bytes: stats.total_failed_bytes,
total_failed_count: stats.total_failed_count,
@@ -298,7 +298,7 @@ fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnaps
}
}
async fn obs_site_replication_stats() -> ReplicationStats {
async fn obs_site_replication_stats() -> ReplicationMetricsSnapshot {
let current_data_transfer_rate = obs_bucket_replication_bandwidth_stats()
.into_iter()
.flatten()
@@ -306,7 +306,7 @@ async fn obs_site_replication_stats() -> ReplicationStats {
.sum::<f64>();
let stats = obs_replication_site_stats_snapshot(current_data_transfer_rate).await;
ReplicationStats {
ReplicationMetricsSnapshot {
average_active_workers: stats.average_active_workers,
average_queued_bytes: stats.average_queued_bytes,
average_queued_count: stats.average_queued_count,
@@ -648,7 +648,7 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
}
/// Collect bucket and target level replication stats from the global replication runtime.
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationMetricsSnapshot> {
obs_bucket_replication_stats_snapshot()
.await
.into_iter()
@@ -662,7 +662,7 @@ pub(crate) async fn collect_bucket_replication_stats_bundle()
}
/// Collect site-level replication stats from the global replication runtime.
pub async fn collect_replication_stats() -> ReplicationStats {
pub async fn collect_replication_stats() -> ReplicationMetricsSnapshot {
obs_site_replication_stats().await
}
crates/policy/AGENTS.md
-1
View File
@@ -23,4 +23,3 @@ Applies to `crates/policy/`.
## Suggested Validation
- `cargo test -p rustfs-policy`
- Full gate before commit: `make pre-commit`
crates/policy/src/policy/policy.rs
+40 -2
View File
@@ -195,7 +195,8 @@ pub struct BucketPolicyArgs<'a> {
#[derive(Serialize, Deserialize, Clone, Default, Debug)]
#[serde(deny_unknown_fields)]
pub struct BucketPolicy {
#[serde(default, rename = "Id", skip_serializing_if = "ID::is_empty")]
// RUSTFS_COMPAT_TODO(rustfs-6339): accept bucket policies persisted with the legacy "ID" key. Remove after migration tooling rewrites every retained legacy bucket policy.
#[serde(default, rename = "Id", alias = "ID", skip_serializing_if = "ID::is_empty")]
pub id: ID,
#[serde(rename = "Version")]
pub version: String,
@@ -2786,7 +2787,7 @@ mod test {
let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse");
// Verify empty fields are omitted
assert!(!parsed.as_object().unwrap().contains_key("ID"), "Empty ID should be omitted");
assert!(parsed.get("Id").is_none(), "Empty ID should be omitted");
let statement = &parsed["Statement"][0];
assert!(!statement.as_object().unwrap().contains_key("Sid"), "Empty Sid should be omitted");
@@ -2809,6 +2810,43 @@ mod test {
assert_eq!(statement["Principal"]["AWS"], "*");
}
#[test]
fn test_bucket_policy_deserializes_legacy_id() {
let legacy_policy = br#"{"ID":"","Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"NotAction":[],"Resource":["arn:aws:s3:::bucket/*"],"NotResource":[],"Condition":{}}]}"#;
let policy: BucketPolicy =
serde_json::from_slice(legacy_policy).expect("bucket policy with legacy ID should deserialize");
assert!(policy.id.is_empty());
policy.is_valid().expect("legacy bucket policy should remain valid");
let policy: BucketPolicy = serde_json::from_str(r#"{"ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#)
.expect("non-empty legacy ID should deserialize");
assert_eq!(policy.id.0, "legacy-policy");
let serialized = serde_json::to_value(&policy).expect("bucket policy should serialize");
assert_eq!(serialized["Id"], "legacy-policy");
assert!(serialized.get("ID").is_none(), "legacy ID spelling should not be serialized");
}
#[test]
fn test_bucket_policy_legacy_id_alias_remains_strict() {
let unknown_field = r#"{"Version":"2012-10-17","Statement":[],"Unexpected":true}"#;
let error =
serde_json::from_str::<BucketPolicy>(unknown_field).expect_err("unrelated unknown fields should remain rejected");
assert!(
error.to_string().contains("unknown field `Unexpected`"),
"unexpected deserialization error: {error}"
);
let duplicate_id = r#"{"Id":"current-policy","ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#;
let error = serde_json::from_str::<BucketPolicy>(duplicate_id)
.expect_err("canonical and legacy ID fields should not be accepted together");
assert!(
error.to_string().contains("duplicate field `Id`"),
"unexpected deserialization error: {error}"
);
}
#[test]
fn test_existing_object_tag_condition_helpers() {
let identity_policy = Policy::parse_config(

Some files were not shown because too many files have changed in this diff Show More