Compare commits

..

2 Commits

Author SHA1 Message Date
overtrue 30c6f8a9ce fix(data-usage): make empty checks exhaustive 2026-07-29 16:11:42 +08:00
overtrue c77c9875c6 fix(data-usage): preserve nonempty replication stats 2026-07-29 15:17:50 +08:00
164 changed files with 4090 additions and 16468 deletions
+2 -36
View File
@@ -173,39 +173,8 @@ jobs:
run: cargo clippy --all-targets -- -D warnings
- name: Run nextest tests
env:
# Three concurrent workspace test links saturate the self-hosted
# runner's overlay I/O and can wedge Cargo until the 75m timeout.
CARGO_BUILD_JOBS: "2"
run: |
mkdir -p artifacts/test-and-lint
# Evidence sampler for issue #5394: the post-mortem pgrep below runs
# only after `timeout` has already TERM'd the whole cargo process
# group, so it cannot name a wedged process. Sample system and
# process state every 60s instead; the last samples before the
# timeout show what was stuck (rustc, linker, build script, memory
# pressure, ...). The log rides along in the existing artifact.
(
while true; do
{
echo "=== $(date --utc --iso-8601=seconds)"
echo "--- load"; cat /proc/loadavg
echo "--- psi"; grep -H . /proc/pressure/* 2>/dev/null || true
echo "--- mem"; free -m
echo "--- disk"; df -h / /home/runner 2>/dev/null || df -h /
echo "--- top-rss"
ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss | head -15
echo "--- build/test processes"
ps -eo pid,ppid,stat,etime,rss,pcpu,args | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true
echo "--- d-state (uninterruptible IO)"
ps -eo pid,stat,etime,args | awk 'NR > 1 && $2 ~ /D/' || true
echo
} >> artifacts/test-and-lint/sampler.log 2>&1 || true
sleep 60
done
) &
sampler_pid=$!
trap 'kill "${sampler_pid}" 2>/dev/null || true' EXIT
set +e
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
cargo nextest run --profile ci --all --exclude e2e_test \
@@ -219,9 +188,6 @@ jobs:
echo
echo "Remaining test-related processes:"
pgrep -af 'cargo|nextest|target/.*/deps/' || true
echo
echo "Kernel OOM / kill events:"
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
} > artifacts/test-and-lint/nextest-diagnostics.txt
exit "${status}"
@@ -406,7 +372,7 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build debug binary
run: cargo build -p rustfs --bins --features e2e-test-hooks
run: cargo build -p rustfs --bins
- name: Upload debug binary
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
@@ -436,7 +402,7 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build debug binary with rio-v2
run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
run: cargo build -p rustfs --bins --features rio-v2
- name: Upload debug binary
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
Generated
+18 -16
View File
@@ -3671,13 +3671,12 @@ dependencies = [
"flatbuffers",
"flate2",
"futures",
"hex",
"http 1.4.2",
"http-body-util",
"hyper",
"hyper-util",
"local-ip-address",
"md-5 0.11.0",
"md5",
"opentelemetry-proto",
"prost 0.14.4",
"rand 0.10.2",
@@ -5645,9 +5644,9 @@ dependencies = [
[[package]]
name = "lazy-regex"
version = "3.6.1"
version = "3.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4994ba703f78b083e2f7946dac9251abd83fd43a0365f030e99b69be5b4b9ef9"
checksum = "6bae91019476d3ec7147de9aa291cadb6d870abf2f3015d2da73a90325ac1496"
dependencies = [
"lazy-regex-proc_macros",
"once_cell",
@@ -5656,9 +5655,9 @@ dependencies = [
[[package]]
name = "lazy-regex-proc_macros"
version = "3.6.1"
version = "3.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd97232314824e6dbef1918a871bb93f51070455e3715bf26e19a6d01aa977a0"
checksum = "4de9c1e1439d8b7b3061b2d209809f447ca33241733d9a3c01eabf2dc8d94358"
dependencies = [
"proc-macro2",
"quote",
@@ -7237,6 +7236,12 @@ dependencies = [
"path-dedot",
]
[[package]]
name = "path-clean"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
[[package]]
name = "path-dedot"
version = "4.0.1"
@@ -8885,7 +8890,7 @@ dependencies = [
"libmimalloc-sys",
"libsystemd",
"matchit 0.9.2",
"md-5 0.11.0",
"md5",
"metrics",
"metrics-util",
"mimalloc",
@@ -9090,6 +9095,7 @@ name = "rustfs-data-usage"
version = "1.0.0-beta.11"
dependencies = [
"async-trait",
"path-clean",
"rmp-serde",
"rustfs-filemeta",
"serde",
@@ -9099,6 +9105,7 @@ dependencies = [
name = "rustfs-ecstore"
version = "1.0.0-beta.11"
dependencies = [
"aes-gcm",
"arc-swap",
"async-channel",
"async-recursion",
@@ -9114,6 +9121,7 @@ dependencies = [
"byteorder",
"bytes",
"bytesize",
"chacha20poly1305",
"chrono",
"criterion",
"enumset",
@@ -9166,6 +9174,7 @@ dependencies = [
"rustfs-erasure-codec",
"rustfs-filemeta",
"rustfs-io-metrics",
"rustfs-kms",
"rustfs-lifecycle",
"rustfs-lock",
"rustfs-madmin",
@@ -9202,14 +9211,11 @@ dependencies = [
"tonic",
"tower",
"tracing",
"tracing-core",
"tracing-opentelemetry",
"tracing-subscriber",
"url",
"urlencoding",
"uuid",
"winapi-util",
"windows-sys 0.61.2",
"xxhash-rust",
]
@@ -9443,10 +9449,9 @@ dependencies = [
"async-trait",
"base64 0.23.0",
"chacha20poly1305",
"hex",
"insta",
"jiff",
"md-5 0.11.0",
"md5",
"moka",
"rand 0.10.2",
"reqwest",
@@ -9455,7 +9460,6 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.11.0",
"subtle",
"temp-env",
"tempfile",
"thiserror 2.0.19",
@@ -9713,7 +9717,7 @@ dependencies = [
"hyper-util",
"ipnetwork",
"libunftp",
"md-5 0.11.0",
"md5",
"percent-encoding",
"proptest",
"quick-xml",
@@ -10124,7 +10128,6 @@ dependencies = [
"bytes",
"convert_case 0.11.0",
"crc-fast",
"criterion",
"flate2",
"futures",
"hex-simd",
@@ -11826,7 +11829,6 @@ dependencies = [
"futures-util",
"libc",
"pin-project-lite",
"slab",
"tokio",
]
+1 -3
View File
@@ -265,6 +265,7 @@ memmap2 = "0.9.11"
lz4 = "1.28.1"
matchit = "0.9.2"
md-5 = "0.11.0"
md5 = "0.8.1"
mime_guess = "2.0.5"
moka = { version = "0.12.15" }
netif = "0.1.6"
@@ -303,7 +304,6 @@ test-case = "3.3.1"
thiserror = "2.0.19"
tracing = { version = "0.1.44" }
tracing-appender = "0.2.5"
tracing-core = "0.1.36"
tracing-error = "0.2.1"
tracing-opentelemetry = { version = "0.33" }
tracing-subscriber = { version = "0.3.23" }
@@ -314,9 +314,7 @@ uuid = { version = "1.24.0" }
vaultrs = { version = "0.8.0" }
tar = "0.4.46"
walkdir = "2.5.0"
winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.18" }
zip = "8.6.0"
zstd = "0.13.3"
+33 -145
View File
@@ -1114,8 +1114,6 @@ pub struct ScannerLastMinute {
pub struct ScannerMetricsReport {
pub collected_at: DateTime<Utc>,
pub current_cycle: u64,
#[serde(default)]
pub current_cycle_active: bool,
pub current_started: DateTime<Utc>,
pub cycles_completed_at: Vec<DateTime<Utc>>,
pub ongoing_buckets: usize,
@@ -2053,7 +2051,7 @@ impl Metrics {
pub fn record_scanner_transition_failed(&self, count: u64) {
self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
self.record_scanner_source_failed(ScannerWorkSource::Lifecycle, count);
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
self.record_last_cycle_scanner_source_work(ScannerWorkSource::Lifecycle, ScannerSourceWorkUpdate::failed(count));
}
}
@@ -2338,21 +2336,6 @@ impl Metrics {
*self.cycle_info.write().await = cycle;
}
/// Publish a scanner cycle and its work-accounting baseline as one state transition.
pub async fn start_scan_cycle_work_with_cycle(&self, cycle: CurrentCycle) -> ScanCycleWorkSnapshot {
let mut current_cycle = self.cycle_info.write().await;
let snapshot = self.start_scan_cycle_work();
*current_cycle = Some(cycle);
snapshot
}
/// Publish the completed work snapshot and idle cycle state as one state transition.
pub async fn finish_scan_cycle_work_with_cycle(&self, start: ScanCycleWorkSnapshot, cycle: CurrentCycle) {
let mut current_cycle = self.cycle_info.write().await;
self.finish_scan_cycle_work(start);
*current_cycle = Some(cycle);
}
/// Read the current cycle record.
pub async fn get_cycle(&self) -> Option<CurrentCycle> {
self.cycle_info.read().await.clone()
@@ -2481,7 +2464,7 @@ impl Metrics {
&self.current_scan_cycle_replication_repair_work_start,
&replication_repair_snapshot,
);
self.current_scan_cycle_work_active.store(true, Ordering::Release);
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
snapshot
}
@@ -2493,11 +2476,11 @@ impl Metrics {
self.record_scan_cycle_work(work);
self.record_scan_cycle_source_work(&source_work);
self.record_scan_cycle_replication_repair_work(&replication_repair_work);
self.current_scan_cycle_work_active.store(false, Ordering::Release);
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
}
pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
return false;
}
@@ -2763,41 +2746,13 @@ impl Metrics {
pub async fn report(&self) -> ScannerMetricsReport {
let mut m = ScannerMetricsReport::default();
let has_cycle = {
let cycle = self.cycle_info.read().await;
let has_cycle = if let Some(cycle) = cycle.as_ref() {
m.current_cycle = cycle.current;
m.cycles_completed_at = cycle.cycle_completed.clone();
m.current_started = cycle.started;
true
} else {
false
};
m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
if m.current_cycle_active {
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
let current_replication_repair_work =
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
m.current_cycle_objects_scanned = current_work.objects_scanned;
m.current_cycle_directories_scanned = current_work.directories_scanned;
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
m.current_cycle_yield_events = current_work.yield_events;
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
m.current_cycle_ilm_actions = current_work.ilm_actions;
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
m.current_cycle_heal_objects = current_work.heal_objects;
m.current_cycle_replication_checks = current_work.replication_checks;
m.current_cycle_usage_saves = current_work.usage_saves;
m.current_cycle_source_work = self.scanner_source_work_snapshots(&current_source_work);
m.current_cycle_replication_repair =
self.scanner_replication_repair_work_snapshots(&current_replication_repair_work);
}
has_cycle
let has_cycle = if let Some(cycle) = self.get_cycle().await {
m.current_cycle = cycle.current;
m.cycles_completed_at = cycle.cycle_completed;
m.current_started = cycle.started;
true
} else {
false
};
if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
@@ -2838,6 +2793,28 @@ impl Metrics {
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
if self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
let current_replication_repair_work =
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
m.current_cycle_objects_scanned = current_work.objects_scanned;
m.current_cycle_directories_scanned = current_work.directories_scanned;
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
m.current_cycle_yield_events = current_work.yield_events;
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
m.current_cycle_ilm_actions = current_work.ilm_actions;
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
m.current_cycle_heal_objects = current_work.heal_objects;
m.current_cycle_replication_checks = current_work.replication_checks;
m.current_cycle_usage_saves = current_work.usage_saves;
m.current_cycle_source_work = self.scanner_source_work_snapshots(&current_source_work);
m.current_cycle_replication_repair = self.scanner_replication_repair_work_snapshots(&current_replication_repair_work);
}
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
m.last_cycle_result_code = last_cycle_result as u64;
@@ -4165,8 +4142,6 @@ mod tests {
let report = metrics.report().await;
assert!(report.current_cycle_active);
assert_eq!(report.current_cycle, 0);
assert_eq!(report.current_cycle_objects_scanned, 7);
assert_eq!(report.current_cycle_directories_scanned, 3);
assert_eq!(report.current_cycle_bucket_drive_scans, 2);
@@ -4183,8 +4158,6 @@ mod tests {
metrics.finish_scan_cycle_work(start);
let report = metrics.report().await;
assert!(!report.current_cycle_active);
assert_eq!(report.current_cycle, 0);
assert_eq!(report.current_cycle_objects_scanned, 0);
assert_eq!(report.current_cycle_directories_scanned, 0);
assert_eq!(report.current_cycle_bucket_drive_scans, 0);
@@ -4211,91 +4184,6 @@ mod tests {
assert_eq!(report.last_cycle_usage_saves, 2);
}
#[tokio::test]
async fn scan_cycle_activity_and_cycle_state_publish_together() {
let metrics = Metrics::new();
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
let active_cycle = CurrentCycle {
current: 12,
next: 13,
started: cycle_started,
..Default::default()
};
let cycle_state = metrics.cycle_info.read().await;
let mut start_transition = Box::pin(metrics.start_scan_cycle_work_with_cycle(active_cycle));
let waker = std::task::Waker::noop();
let mut context = std::task::Context::from_waker(waker);
assert!(start_transition.as_mut().poll(&mut context).is_pending());
assert!(!metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
drop(cycle_state);
let start = start_transition.await;
let active = metrics.report().await;
assert!(active.current_cycle_active);
assert_eq!(active.current_cycle, 12);
assert_eq!(active.current_started, cycle_started);
let idle_cycle = CurrentCycle {
current: 0,
next: 13,
started: cycle_started,
..Default::default()
};
let cycle_state = metrics.cycle_info.read().await;
let mut finish_transition = Box::pin(metrics.finish_scan_cycle_work_with_cycle(start, idle_cycle));
assert!(finish_transition.as_mut().poll(&mut context).is_pending());
assert!(metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
drop(cycle_state);
finish_transition.await;
let idle = metrics.report().await;
assert!(!idle.current_cycle_active);
assert_eq!(idle.current_cycle, 0);
}
#[tokio::test]
async fn report_keeps_cycle_identity_and_work_in_one_snapshot() {
let metrics = Metrics::new();
let cycle_ten = CurrentCycle {
current: 10,
next: 11,
started: Utc::now() - chrono::Duration::seconds(10),
..Default::default()
};
let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
let paths = metrics.current_paths.write().await;
let mut report = Box::pin(metrics.report());
let waker = std::task::Waker::noop();
let mut context = std::task::Context::from_waker(waker);
assert!(report.as_mut().poll(&mut context).is_pending());
metrics
.finish_scan_cycle_work_with_cycle(cycle_ten_start, CurrentCycle { current: 0, ..cycle_ten })
.await;
let cycle_eleven_start = metrics
.start_scan_cycle_work_with_cycle(CurrentCycle {
current: 11,
next: 12,
started: Utc::now(),
..Default::default()
})
.await;
metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
drop(paths);
let snapshot = report.await;
assert_eq!(snapshot.current_cycle, 10);
assert_eq!(snapshot.current_cycle_objects_scanned, 1);
metrics
.finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
.await;
}
#[tokio::test]
async fn scanner_cycle_ilm_actions_ignore_global_ilm_work() {
let metrics = Metrics::new();
-3
View File
@@ -10,9 +10,6 @@ description = "Shared concurrency contract types for RustFS - workload admission
keywords = ["rustfs", "concurrency", "admission", "backpressure", "workers"]
categories = ["concurrency", "filesystem"]
[lints]
workspace = true
[dependencies]
# Internal crates
rustfs-io-core = { workspace = true }
+8 -30
View File
@@ -158,33 +158,17 @@ pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
/// Require the replay-scoped internode RPC signature after the fleet has converged on it.
///
/// The default keeps v1/v2 peers available during a rolling upgrade. Operators may set this only
/// after `rustfs_system_network_internode_replay_scope_fallback_total` remains zero for a full
/// release window. The node still accepts a v2-authenticated `Ping` carrying an epoch challenge:
/// that narrowly scoped bootstrap lets an upgraded client learn the receiving process epoch and
/// immediately retry with the replay-scoped signature after a peer restart.
pub const ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT: &str = "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT";
pub const DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT: bool = false;
// Compile-time invariant: mixed-version clusters must remain available until operators make the
// observed fallback counter an explicit strictness decision.
const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
/// one-time consumption of authenticated RPC signatures.
/// one-time consumption of body-bound v2 signatures.
///
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
/// the shared secret) — and increments
/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
/// shared secret) — and increments
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
/// counter means this capacity is undersized for the node's peak mutation rate.
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
@@ -370,12 +354,6 @@ mod tests {
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
}
#[test]
fn internode_replay_scope_strict_env_name_is_stable() {
// The fail-open default invariant is asserted at compile time next to the definition.
assert_eq!(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT");
}
#[test]
fn internode_replay_cache_capacity_defaults_and_env_name() {
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
-33
View File
@@ -116,27 +116,6 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
/// Default: bitrot verification is enabled on GetObject reads (do not skip).
pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
/// Request writing the complete remote-tier version state into object metadata.
///
/// This remains ineffective until
/// [`ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED`] is also enabled.
pub const ENV_TIER_REMOTE_VERSION_STATE_WRITE: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE";
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE: bool = false;
/// Operator-attested fleet-wide confirmation for
/// [`ENV_TIER_REMOTE_VERSION_STATE_WRITE`].
///
/// This flag is an operational contract, not automatic capability discovery.
/// Operators may enable it only after every node that can write or read
/// transitioned object metadata supports the remote version-state schema and
/// semantics. Keeping the confirmation separate makes a single-node request or
/// a writer whose local opt-in is removed fail closed.
pub const ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED";
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
// =============================================================================
// Concurrent Request Fix - Timeout and Backpressure Configuration
// =============================================================================
@@ -638,15 +617,3 @@ pub const ENV_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: &str = "RUSTFS_OBJ
/// Default read-ahead disable concurrency threshold: 4.
pub const DEFAULT_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: usize = 4;
#[cfg(test)]
mod remote_version_state_tests {
#[test]
fn remote_version_state_gate_uses_stable_environment_names() {
assert_eq!(super::ENV_TIER_REMOTE_VERSION_STATE_WRITE, "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE");
assert_eq!(
super::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
);
}
}
+1
View File
@@ -29,6 +29,7 @@ workspace = true
[dependencies]
serde = { workspace = true, features = ["derive"] }
path-clean = { workspace = true }
rmp-serde = { workspace = true }
async-trait = { workspace = true }
rustfs-filemeta = { workspace = true }
+6 -94
View File
@@ -12,10 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use path_clean::PathClean;
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, HashSet},
hash::{DefaultHasher, Hash, Hasher},
path::Path,
time::{Duration, SystemTime},
};
@@ -332,7 +334,7 @@ impl<'de> Deserialize<'de> for SizeHistogram {
impl SizeHistogram {
pub fn add(&mut self, size: u64) {
let intervals = [
(0, 1024 - 1), // LESS_THAN_1024_B
(0, 1024), // LESS_THAN_1024_B
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
@@ -360,7 +362,7 @@ impl SizeHistogram {
// the sub-ranges in [1 KiB, 512 KiB).
const ONE_MIB: u64 = 1024 * 1024;
let intervals = [
(0, 1024 - 1), // LESS_THAN_1024_B
(0, 1024), // LESS_THAN_1024_B
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
@@ -1108,39 +1110,9 @@ fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String
}
}
fn clean_data_usage_path(data: &str) -> String {
let rooted = data.starts_with('/');
let mut parts = Vec::new();
for part in data.split('/') {
match part {
"" | "." => {}
".." => {
if parts.last().is_some_and(|last| *last != "..") {
parts.pop();
} else if !rooted {
parts.push(part);
}
}
_ => parts.push(part),
}
}
let clean = parts.join("/");
match (rooted, clean.is_empty()) {
(true, true) => "/".to_string(),
(true, false) => format!("/{clean}"),
(false, true) => ".".to_string(),
(false, false) => clean,
}
}
/// Hash a slash-separated path for data usage caching.
///
/// Cache identifiers are persisted and exchanged across nodes, so their
/// normalization must not depend on the host operating system.
/// Hash a path for data usage caching
pub fn hash_path(data: &str) -> DataUsageHash {
DataUsageHash(clean_data_usage_path(data))
DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
}
impl DataUsageInfo {
@@ -1525,23 +1497,6 @@ mod tests {
buckets_count: u64,
}
#[test]
fn hash_path_uses_portable_slash_semantics() {
for (input, expected) in [
("", "."),
(".", "."),
("/", "/"),
("//bucket///prefix/", "/bucket/prefix"),
("bucket/./prefix//object", "bucket/prefix/object"),
("bucket/a/../b", "bucket/b"),
("../bucket/..", ".."),
("/../../bucket", "/bucket"),
("bucket\\prefix/object", "bucket\\prefix/object"),
] {
assert_eq!(hash_path(input).key(), expected, "unexpected portable cache key for {input:?}");
}
}
#[test]
fn completeness_marker_is_additive_for_legacy_named_readers() {
let current = DataUsageInfo {
@@ -1646,49 +1601,6 @@ mod tests {
assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1);
}
#[test]
fn test_size_histogram_classifies_adjacent_boundaries_once() {
let cases = [
(1023, 0),
(1024, 1),
(64 * 1024 - 1, 1),
(64 * 1024, 2),
(256 * 1024 - 1, 2),
(256 * 1024, 3),
(512 * 1024 - 1, 3),
(512 * 1024, 4),
(1024 * 1024 - 1, 4),
(1024 * 1024, 6),
(10 * 1024 * 1024 - 1, 6),
(10 * 1024 * 1024, 7),
(64 * 1024 * 1024 - 1, 7),
(64 * 1024 * 1024, 8),
(128 * 1024 * 1024 - 1, 8),
(128 * 1024 * 1024, 9),
(512 * 1024 * 1024 - 1, 9),
(512 * 1024 * 1024, 10),
];
for (size, expected_bucket) in cases {
let mut hist = SizeHistogram::default();
hist.add(size);
assert_eq!(hist.0.iter().sum::<u64>(), 1, "size {size} must have exactly one physical bucket");
assert_eq!(hist.0[expected_bucket], 1, "size {size} must select the expected bucket");
}
}
#[test]
fn test_size_histogram_1024_bytes_contributes_to_compat_rollup() {
let mut hist = SizeHistogram::default();
hist.add(1024);
let map = hist.to_map();
assert_eq!(map["LESS_THAN_1024_B"], 0);
assert_eq!(map["BETWEEN_1024_B_AND_64_KB"], 1);
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], 1);
}
#[test]
fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() {
let mut hist = SizeHistogram::default();
+1 -2
View File
@@ -70,8 +70,7 @@ walkdir.workspace = true
base64 = { workspace = true }
rand = { workspace = true, features = ["serde"] }
chrono = { workspace = true, features = ["serde"] }
hex = { workspace = true }
md-5 = { workspace = true }
md5 = { workspace = true }
opentelemetry-proto = { workspace = true }
prost.workspace = true
sha2 = { workspace = true }
+2 -5
View File
@@ -24,10 +24,9 @@ mod tests {
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine;
use md5::{Digest as Md5Digest, Md5};
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use serial_test::serial;
use sha2::Sha256;
use sha2::{Digest, Sha256};
use tracing::info;
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
@@ -71,9 +70,7 @@ mod tests {
}
fn content_md5_base64(body: &[u8]) -> String {
let mut hasher = Md5::new();
hasher.update(body);
let digest = hasher.finalize();
let digest = md5::compute(body);
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
}
+5 -18
View File
@@ -25,7 +25,6 @@ use hyper::body::Incoming;
use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper_util::rt::{TokioIo, TokioTimer};
use md5::{Digest as Md5Digest, Md5};
use s3s::access::{S3Access, S3AccessContext};
use s3s::auth::SimpleAuth;
use s3s::dto::{
@@ -828,25 +827,13 @@ fn ensure_body_growth(current: usize, added: usize) -> S3Result {
async fn md5_digest(body: Bytes, permit: OwnedSemaphorePermit) -> S3Result<([u8; 16], OwnedSemaphorePermit)> {
if body.len() < 1024 * 1024 {
return Ok((md5_bytes(body), permit));
return Ok((md5::compute(body).0, permit));
}
tokio::task::spawn_blocking(move || (md5_bytes(body), permit))
tokio::task::spawn_blocking(move || (md5::compute(body).0, permit))
.await
.map_err(|error| s3s::s3_error!(InternalError, "MD5 worker failed: {error}"))
}
fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hasher.finalize().into()
}
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
}
fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result {
let total_bytes = state
.total_bytes
@@ -1018,7 +1005,7 @@ impl S3 for FakeBackend {
Some(value) => value,
None => {
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
hex::encode(digest)
format!("{:x}", md5::Digest(digest))
}
};
let version = ObjectVersion {
@@ -1221,7 +1208,7 @@ impl S3 for FakeBackend {
}
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
let e_tag = hex::encode(digest);
let e_tag = format!("{:x}", md5::Digest(digest));
let mut state = lock(&self.store);
let existing_bytes = state
.uploads
@@ -1349,7 +1336,7 @@ impl S3 for FakeBackend {
.collect();
let (body, digests, _body_permits) = assemble_multipart(assembly_parts, total_len, _body_permits).await?;
let part_count = requested.len();
let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{}-{part_count}", md5_hex(digests)));
let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{:x}-{part_count}", md5::compute(digests)));
let version = ObjectVersion {
version_id: upload.version_id.clone(),
body,
@@ -13,9 +13,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Cross-process replay / tamper acceptance for internode NodeService RPC
//! signatures (<https://github.com/rustfs/backlog/issues/1327>,
//! <https://github.com/rustfs/backlog/issues/1542>).
//! Cross-process replay / tamper acceptance for the internode NodeService v2 RPC
//! signature (<https://github.com/rustfs/backlog/issues/1327>).
//!
//! # Why this exists on top of the in-process tests
//!
@@ -79,8 +78,6 @@
//! | mixed version: legacy-only still served, not blocked | [`legacy_only_signature_is_accepted_in_default_posture`] |
//! | strict flip closes the signature downgrade | [`signature_strict_rejects_legacy_only_downgrade`] |
//! | strict flip closes the body-digest downgrade, incl. v1 | [`body_digest_strict_rejects_digestless_mutation`] |
//! | replay scope binds every RPC and rejects restart replay | [`replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e`] |
//! | strict replay scope allows only Ping bootstrap before v3 | [`replay_scope_strict_requires_v3_after_ping_bootstrap_e2e`] |
//!
//! Two acceptance items are deliberately left to the in-process tests. A stale
//! timestamp cannot be forged from outside — it is inside the HMAC — so
@@ -91,20 +88,18 @@
use crate::common::{RustFSTestEnvironment, init_logging};
use crate::storage_api::internode_rpc_signature::{
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
};
use http::{HeaderMap, Method};
use rustfs_config::{
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
ENV_INTERNODE_RPC_SIGNATURE_STRICT,
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_SIGNATURE_STRICT,
};
use rustfs_protos::canonical_make_volume_request_body;
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse};
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::error::Error;
use tonic::{Code, Request, Response, Status};
use tonic::{Code, Request, Status};
use uuid::Uuid;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
@@ -120,14 +115,12 @@ const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
/// clears authentication stops harmlessly at `find_disk`.
const ABSENT_DISK: &str = "/nonexistent/rustfs-signature-e2e-disk";
/// Wire names of the v2 and replay-scope headers these black-box tests edit. They are
/// `pub(crate)` in ecstore, so they are repeated here rather than imported.
/// [`overwrite_header`] asserts the header it replaces was actually present, which turns a
/// rename into a loud failure instead of silently reducing an attack to a no-op.
/// Wire names of the two v2 headers these tests edit. They are `pub(crate)` in
/// ecstore, so they are repeated here rather than imported — [`overwrite_header`]
/// asserts the header it replaces was actually present, which turns a rename
/// into a loud failure instead of silently reducing an attack to a no-op.
const CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
const NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
const BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
/// gRPC service name carried in the signed scope, i.e. `TONIC_RPC_PREFIX`
/// without its leading `/`.
@@ -162,28 +155,19 @@ fn align_rpc_secret_with_server() {
///
/// Uses the no-cleanup spawn so a `pkill` pattern cannot reap servers belonging
/// to other tests running in the same binary.
fn server_env(extra_env: &[(&'static str, &'static str)]) -> Vec<(&'static str, &'static str)> {
async fn start_server(extra_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
let mut env = RustFSTestEnvironment::new().await?;
let mut child_env = vec![
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "false"),
(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "false"),
(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "false"),
(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "1048576"),
];
child_env.extend_from_slice(extra_env);
child_env
}
async fn start_server_with_env(child_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_without_cleanup_with_env(child_env).await?;
env.start_rustfs_server_without_cleanup_with_env(&child_env).await?;
Ok(env)
}
async fn start_server(extra_env: &[(&'static str, &'static str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
start_server_with_env(&server_env(extra_env)).await
}
/// Stop the child and drop the cached gRPC channel for its address.
///
/// `node_service_time_out_client_no_auth` memoises channels in a process-global
@@ -254,83 +238,12 @@ fn overwrite_header(headers: &mut HeaderMap, name: &'static str, value: &str) {
/// and nothing else — no interceptor adds or rewrites auth metadata, so the
/// bytes on the wire are the ones the test chose.
async fn call_make_volume(url: &str, request: MakeVolumeRequest, headers: HeaderMap) -> Result<MakeVolumeResponse, Status> {
call_make_volume_response(url, request, headers)
.await
.map(Response::into_inner)
}
async fn call_make_volume_response(
url: &str,
request: MakeVolumeRequest,
headers: HeaderMap,
) -> Result<Response<MakeVolumeResponse>, Status> {
let mut client = node_service_time_out_client_no_auth(&url.to_string())
.await
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
let mut rpc_request = Request::new(request);
rpc_request.metadata_mut().as_mut().extend(headers);
client.make_volume(rpc_request).await
}
async fn call_ping_response(url: &str, headers: HeaderMap) -> Result<Response<PingResponse>, Status> {
let mut client = node_service_time_out_client_no_auth(&url.to_string())
.await
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
let mut rpc_request = Request::new(PingRequest {
version: 1,
body: bytes::Bytes::new(),
});
rpc_request.metadata_mut().as_mut().extend(headers);
client.ping(rpc_request).await
}
fn attach_boot_epoch_challenge(headers: &mut HeaderMap) -> Uuid {
let challenge = Uuid::new_v4();
headers.insert(
BOOT_EPOCH_CHALLENGE_HEADER,
challenge.to_string().parse().expect("UUID must be a valid header value"),
);
challenge
}
fn mint_replay_scope_headers(audience: &str, path: &str, content_sha256: &str, boot_epoch: Uuid) -> HeaderMap {
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(content_sha256));
let timestamp = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 headers must carry a timestamp")
.to_string();
headers.extend(
gen_tonic_replay_scope_headers(audience, path, &timestamp, content_sha256, boot_epoch)
.expect("replay-scope headers must mint with the aligned RPC secret"),
);
headers
}
async fn learn_boot_epoch_from_make_volume(url: &str, audience: &str) -> Uuid {
let request = make_volume_request("signature-e2e-epoch-bootstrap");
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(&canonical_digest(&request)));
let challenge = attach_boot_epoch_challenge(&mut headers);
let response = call_make_volume_response(url, request, headers)
.await
.expect("v2 request with epoch challenge must clear default authentication");
let boot_epoch = verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
.expect("server must HMAC-authenticate the advertised boot epoch");
assert_authenticated(
Ok(response.into_inner()),
"a v2 epoch-challenge request in the default replay-scope posture",
);
boot_epoch
}
async fn learn_boot_epoch_from_ping(url: &str, audience: &str) -> Uuid {
let mut headers = mint_v2_headers(audience, "Ping", None);
let challenge = attach_boot_epoch_challenge(&mut headers);
let response = call_ping_response(url, headers)
.await
.expect("v2 Ping with an epoch challenge must bootstrap strict replay scope");
verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
.expect("strict replay-scope Ping must return a valid boot epoch proof")
client.make_volume(rpc_request).await.map(|response| response.into_inner())
}
/// Assert a call cleared authentication.
@@ -419,122 +332,6 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
Ok(())
}
/// A replay-scoped signature is usable exactly once against the exact gRPC path and the server
/// process epoch that minted it. This crosses the child-process boundary twice: the HMAC-protected
/// epoch is learned from a real response, then the same server is restarted in place to prove its
/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
#[tokio::test]
#[serial]
async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
init_logging();
align_rpc_secret_with_server();
let child_env = server_env(&[]);
let mut env = start_server_with_env(&child_env).await?;
let url = env.url.clone();
let audience = audience_of(&env);
let boot_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
let request = make_volume_request("replay-scope-e2e-once");
let captured = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
boot_epoch,
);
assert_authenticated(
call_make_volume(&url, request.clone(), captured.clone()).await,
"the first replay-scoped mutation delivery",
);
assert_rejected(
call_make_volume(&url, request.clone(), captured).await,
Code::Unauthenticated,
None,
"the same replay-scoped mutation delivered twice",
);
let transplanted =
mint_replay_scope_headers(&audience, &format!("{TONIC_RPC_PREFIX}/Ping"), &canonical_digest(&request), boot_epoch);
assert_rejected(
call_make_volume(&url, request.clone(), transplanted).await,
Code::Unauthenticated,
None,
"a replay-scoped Ping signature transplanted onto MakeVolume",
);
let stale_epoch = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
boot_epoch,
);
env.restart_server_preserving_data(Vec::new(), &child_env).await?;
rustfs_protos::evict_failed_connection(&url).await;
assert_rejected(
call_make_volume(&url, request.clone(), stale_epoch).await,
Code::Unauthenticated,
None,
"a replay-scoped signature captured before the receiving process restart",
);
let restarted_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
assert_ne!(boot_epoch, restarted_epoch, "a restarted child process must advertise a new boot epoch");
let fresh_epoch = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
restarted_epoch,
);
assert_authenticated(
call_make_volume(&url, request, fresh_epoch).await,
"a replay-scoped mutation signed with the replacement process epoch",
);
stop_server(env, &url).await;
Ok(())
}
/// Strict replay scope leaves one authenticated v2 bootstrap: `Ping` carrying a fresh challenge.
/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
#[tokio::test]
#[serial]
async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
init_logging();
align_rpc_secret_with_server();
let env = start_server(&[(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "true")]).await?;
let url = env.url.clone();
let audience = audience_of(&env);
let v2_request = make_volume_request("replay-scope-e2e-strict-v2");
assert_rejected(
call_make_volume(
&url,
v2_request.clone(),
mint_v2_headers(&audience, "MakeVolume", Some(&canonical_digest(&v2_request))),
)
.await,
Code::Unauthenticated,
None,
"a v2 mutation after replay-scope strictness is enabled",
);
let boot_epoch = learn_boot_epoch_from_ping(&url, &audience).await;
let request = make_volume_request("replay-scope-e2e-strict-v3");
let replay_scoped = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
boot_epoch,
);
assert_authenticated(
call_make_volume(&url, request, replay_scoped).await,
"a replay-scoped mutation after Ping bootstrap under strict replay scope",
);
stop_server(env, &url).await;
Ok(())
}
/// Baseline: correctly signed mutations are accepted, both with and without a
/// body digest.
///
+1 -4
View File
@@ -30,7 +30,6 @@ use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
use http::header::{CONTENT_TYPE, HOST};
use md5::{Digest as Md5Digest, Md5};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
@@ -69,9 +68,7 @@ pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
}
pub fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
BASE64.encode(hasher.finalize())
BASE64.encode(md5::compute(key).0)
}
pub async fn kms_admin_request(
@@ -25,18 +25,12 @@ use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
use crate::common::{TEST_BUCKET, init_logging};
use aws_sdk_s3::types::ServerSideEncryption;
use base64::Engine;
use md5::{Digest as Md5Digest, Md5};
use md5::compute;
use serial_test::serial;
use std::sync::Arc;
use tokio::sync::Semaphore;
use tracing::{info, warn};
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
}
/// Test encryption of zero-byte files (empty files)
#[tokio::test]
#[serial]
@@ -300,7 +294,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
info!("🔍 Testing invalid SSE-C key length");
let invalid_short_key = "short"; // Too short
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
let invalid_key_md5 = md5_hex(invalid_short_key);
let invalid_key_md5 = format!("{:x}", compute(invalid_short_key));
let invalid_key_result = s3_client
.put_object()
+2 -11
View File
@@ -26,7 +26,6 @@ use chrono::{Duration as ChronoDuration, Utc};
use flate2::{Compression, write::GzEncoder};
use http::HeaderValue;
use http::header::{CONTENT_TYPE, HOST};
use md5::{Digest as Md5Digest, Md5};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
@@ -51,15 +50,7 @@ fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
}
fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
base64::engine::general_purpose::STANDARD.encode(hasher.finalize())
}
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
base64::engine::general_purpose::STANDARD.encode(md5::compute(key).0)
}
/// Env var consumed by the local SSE-S3 DEK provider when KMS is not configured.
@@ -5673,7 +5664,7 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
client.create_bucket().bucket(bucket).send().await?;
let archive = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await;
let expected_etag = format!("\"{}\"", md5_hex(&archive));
let expected_etag = format!("\"{:x}\"", md5::compute(&archive));
let response = client
.put_object()
@@ -21,22 +21,18 @@
//! function, never as an S3 event sink.
//!
//! Coverage:
//! * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct
//! * PUT / multipart-complete / DELETE each deliver one event with the correct
//! eventName, bucket, key, versionId and eTag.
//! * prefix/suffix filters drop non-matching keys (rule-engine gate).
//! * an event queued while the target endpoint is unreachable is redelivered
//! from the on-disk store once the endpoint recovers (store-and-forward).
//! * responseElements and the S3 response use the canonical request ID while
//! requestParameters preserve a conflicting client-supplied value.
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::operation::RequestId;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, Event, FilterRule, FilterRuleName,
NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Event, FilterRule, FilterRuleName,
NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter, VersioningConfiguration,
};
use http::header::{CONTENT_TYPE, HOST};
use local_ip_address::local_ip;
@@ -44,12 +40,10 @@ use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
use s3s::Body;
use serde_json::Value;
use serial_test::serial;
use std::error::Error;
use std::io::Cursor;
use std::path::Path;
use std::sync::{
Arc, Once,
@@ -69,8 +63,6 @@ type BoxError = Box<dyn Error + Send + Sync>;
/// for target lookup (see `process_queue_configurations`), so the region is
/// nominal, but it must be present for `ARN::parse` to succeed.
const NOTIFY_REGION: &str = "us-east-1";
const CLIENT_REQUEST_ID: &str = "client-supplied-request-id";
const CLIENT_AMZ_REQUEST_ID: &str = "client-supplied-amz-request-id";
/// Webhook targets are registered as `TargetID { id: <name>, name: "webhook" }`,
/// so the ARN a notification rule references is
@@ -587,36 +579,6 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
value.map(|e| e.trim_matches('"').to_string())
}
fn assert_conflicting_request_id_correlation(record: &Value, server_request_id: &str) {
assert_eq!(
record["requestParameters"][REQUEST_ID_HEADER].as_str(),
Some(CLIENT_REQUEST_ID),
"notification request parameters should retain the actual client header: {record}"
);
assert_eq!(
record["requestParameters"][AMZ_REQUEST_ID].as_str(),
Some(CLIENT_AMZ_REQUEST_ID),
"notification request parameters should retain the actual client header: {record}"
);
assert_eq!(
record["responseElements"][AMZ_REQUEST_ID].as_str(),
Some(server_request_id),
"notification response elements should use the canonical request ID: {record}"
);
}
fn assert_generated_request_id_correlation(record: &Value, request_id: &str) {
assert!(
record["requestParameters"][AMZ_REQUEST_ID].is_null(),
"notification request parameters must not invent a client request header: {record}"
);
assert_eq!(
record["responseElements"][AMZ_REQUEST_ID].as_str(),
Some(request_id),
"notification response elements should match the S3 response request ID: {record}"
);
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
@@ -709,17 +671,8 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
.bucket(bucket)
.key(put_key)
.body(ByteStream::from_static(b"peri-1 put body"))
.customize()
.mutate_request(|request| {
request.headers_mut().insert(REQUEST_ID_HEADER, CLIENT_REQUEST_ID);
request.headers_mut().insert(AMZ_REQUEST_ID, CLIENT_AMZ_REQUEST_ID);
})
.send()
.await?;
let put_request_id = put.request_id().ok_or("PUT response missing request ID")?.to_owned();
assert!(uuid::Uuid::parse_str(&put_request_id).is_ok());
assert_ne!(put_request_id, CLIENT_REQUEST_ID);
assert_ne!(put_request_id, CLIENT_AMZ_REQUEST_ID);
let put_version = put
.version_id()
.ok_or("PUT response missing versionId (versioning not enabled?)")?;
@@ -739,7 +692,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
"record eventName: {record}"
);
assert_eq!(record["s3"]["bucket"]["name"].as_str(), Some(bucket), "bucket in event: {record}");
assert_conflicting_request_id_correlation(record, &put_request_id);
assert_eq!(object["versionId"].as_str(), Some(put_version), "versionId in event: {object}");
assert_eq!(
trimmed_etag(object["eTag"].as_str()),
@@ -792,35 +744,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
"multipart eTag in event: {mp_record}"
);
// --- Snowball extract: direct notification path keeps response correlation
let snowball_key = "uploads/snowball.dat";
let snowball_body = b"snowball notification body";
let mut archive_builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
let mut archive_header = tokio_tar::Header::new_gnu();
archive_header.set_size(u64::try_from(snowball_body.len()).expect("snowball fixture length should fit in u64"));
archive_header.set_mode(0o644);
archive_header.set_cksum();
archive_builder
.append_data(&mut archive_header, snowball_key, Cursor::new(snowball_body))
.await?;
let archive = archive_builder.into_inner().await?.into_inner();
let snowball = client
.put_object()
.bucket(bucket)
.key("snowball-fixture.tar")
.body(ByteStream::from(archive))
.customize()
.mutate_request(|request| {
request.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
})
.send()
.await?;
let snowball_request_id = snowball.request_id().ok_or("Snowball response missing request ID")?;
let snowball_event = wait_for_event(&mut rx, snowball_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
let snowball_record = &snowball_event["Records"][0];
assert_generated_request_id_correlation(snowball_record, snowball_request_id);
// --- Filter: non-matching prefix and suffix must never be delivered ------
let wrong_prefix = "logs/report.dat"; // right suffix, wrong prefix
let wrong_suffix = "uploads/report.txt"; // right prefix, wrong suffix
@@ -849,32 +772,7 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
"wrong-suffix key {wrong_suffix} bypassed the filter; saw {seen:?}"
);
// --- DeleteObjects: direct notification path keeps response correlation --
let delete_many_key = "uploads/delete-many.dat";
client
.put_object()
.bucket(bucket)
.key(delete_many_key)
.body(ByteStream::from_static(b"delete objects notification body"))
.send()
.await?;
wait_for_event(&mut rx, delete_many_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
let delete_many = client
.delete_objects()
.bucket(bucket)
.delete(
Delete::builder()
.objects(ObjectIdentifier::builder().key(delete_many_key).build()?)
.build()?,
)
.send()
.await?;
let delete_many_request_id = delete_many.request_id().ok_or("DeleteObjects response missing request ID")?;
let delete_many_event = wait_for_event(&mut rx, delete_many_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
assert_generated_request_id_correlation(&delete_many_event["Records"][0], delete_many_request_id);
// --- DeleteObject on a versioned bucket: delete-marker version ----------
// --- DELETE on a versioned bucket: ObjectRemoved:* with delete-marker version
let delete = client.delete_object().bucket(bucket).key(put_key).send().await?;
let removed = wait_for_event(&mut rx, put_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
let removed_record = &removed["Records"][0];
@@ -24,7 +24,7 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
use tonic::Request;
use tracing::{info, warn};
use crate::storage_api::grpc_lock::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
use crate::storage_api::grpc_lock::{TonicInterceptor, node_service_time_out_client_no_auth};
/// gRPC lock client without authentication for testing
/// Similar to RemoteClient but uses no_auth client
@@ -42,7 +42,7 @@ impl GrpcLockClient {
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
tonic::service::interceptor::InterceptedService<AuthenticatedChannel, TonicInterceptor>,
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, TonicInterceptor>,
>,
> {
node_service_time_out_client_no_auth(&self.addr)
@@ -23,8 +23,7 @@ use rustfs_protos::{
proto_gen::node_service::{
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
SnapshotLeaseResponse, node_service_server::NodeService,
node_service_server::NodeService,
},
};
use std::pin::Pin;
@@ -105,27 +104,6 @@ impl NodeService for MinimalLockNodeService {
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
}
async fn acquire_snapshot_lease(
&self,
_request: Request<SnapshotLeaseRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
}
async fn renew_snapshot_lease(
&self,
_request: Request<SnapshotLeaseRenewRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
}
async fn release_snapshot_lease(
&self,
_request: Request<SnapshotLeaseReleaseRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
}
async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: LockRequest = match serde_json::from_str(&request.args) {
+2 -10
View File
@@ -95,7 +95,6 @@ const MANUAL_ASYNC_PARALLEL_OBJECTS: usize = 64;
const MANUAL_ACTIVE_CANCEL_OBJECTS: usize = 512;
const MANUAL_RESTART_CANCEL_OBJECTS: usize = 512;
const MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT: StdDuration = StdDuration::from_secs(15);
const MANUAL_TRANSITION_CANCEL_BARRIER_ENV: &str = "RUSTFS_E2E_MANUAL_TRANSITION_CANCEL_BARRIER";
const MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT: StdDuration = StdDuration::from_secs(90);
const MANUAL_RESTART_RECOVERY_TIMEOUT: StdDuration = StdDuration::from_secs(80);
const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
@@ -1685,15 +1684,8 @@ async fn test_manual_transition_async_active_cancel_reports_terminal_cancelled()
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
let mut hot = RustFSTestEnvironment::new().await?;
hot.start_rustfs_server_with_env(
vec![],
&[
("RUSTFS_SCANNER_ENABLED", "false"),
("RUSTFS_SCANNER_CYCLE", "3600"),
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
],
)
.await?;
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
.await?;
let hot_client = hot.create_s3_client();
add_rustfs_tier(&hot, &cold).await?;
+1 -4
View File
@@ -22,7 +22,6 @@ use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine;
use md5::{Digest as Md5Digest, Md5};
use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
@@ -103,12 +102,10 @@ impl Intercept for ResponseHeaderCapture {
fn customer_key(byte: u8) -> CustomerKey {
let raw = [byte; 32];
let mut hasher = Md5::new();
hasher.update(raw);
CustomerKey {
raw: String::from_utf8_lossy(&raw).into_owned(),
encoded: base64::engine::general_purpose::STANDARD.encode(raw),
md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()),
md5: base64::engine::general_purpose::STANDARD.encode(md5::compute(raw).0),
}
}
+4 -8
View File
@@ -16,12 +16,9 @@
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
verify_tonic_boot_epoch_response,
};
pub(crate) use rustfs_ecstore::api::rpc::{TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers};
pub(crate) use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
@@ -33,7 +30,7 @@ pub(crate) mod node_interact {
}
pub(crate) mod grpc_lock {
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
pub(crate) use super::{TonicInterceptor, node_service_time_out_client_no_auth};
}
/// Signing/transport surface used by the cross-process internode RPC signature
@@ -43,8 +40,7 @@ pub(crate) mod grpc_lock {
#[cfg(test)]
pub(crate) mod internode_rpc_signature {
pub(crate) use super::{
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
};
}
+3 -7
View File
@@ -57,6 +57,7 @@ rustfs-policy.workspace = true
rustfs-protos.workspace = true
rustfs-replication.workspace = true
rustfs-lifecycle.workspace = true
rustfs-kms.workspace = true
rustfs-s3-types = { workspace = true }
rustfs-data-usage.workspace = true
rustfs-object-capacity.workspace = true
@@ -123,6 +124,8 @@ libc.workspace = true
rustix = { workspace = true, features = ["process", "fs"] }
rustfs-madmin.workspace = true
reqwest = { workspace = true }
aes-gcm = { workspace = true, features = ["rand_core"] }
chacha20poly1305.workspace = true
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
urlencoding = { workspace = true }
smallvec = { workspace = true, features = ["serde"] }
@@ -150,18 +153,11 @@ metrics = { workspace = true }
[target.'cfg(target_os = "linux")'.dependencies]
rustfs-uring = "0.2.1"
[target.'cfg(windows)'.dependencies]
winapi-util.workspace = true
windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }
[dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
criterion = { workspace = true, features = ["html_reports"] }
temp-env = { workspace = true, features = ["async_closure"] }
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
# Only for `pin_callsite_interest_for_test`, which registers a `NoSubscriber`
# dispatcher to keep tracing's process-global callsite-interest cache honest.
tracing-core = { workspace = true }
serial_test = { workspace = true }
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
proptest = "1"
+15 -29
View File
@@ -67,14 +67,6 @@ pub mod bucket {
};
}
pub mod transition_transaction {
pub use crate::bucket::lifecycle::transition_transaction::{
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
inspect_transition_transaction_for_operator,
};
}
pub mod evaluator {
pub use crate::bucket::lifecycle::evaluator::Evaluator;
}
@@ -130,13 +122,13 @@ pub mod bucket {
pub mod metadata_sys {
pub use crate::bucket::metadata_sys::{
BucketMetadataSys, acquire_bucket_metadata_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
BucketMetadataSys, acquire_bucket_targets_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
update, update_bucket_targets_under_transaction_lock, update_config_with, update_under_transaction_lock,
init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
update_bucket_targets_under_transaction_lock, update_config_with,
};
}
@@ -313,8 +305,7 @@ pub mod disk {
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
validate_batch_read_version_item_count,
STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
};
pub use bytes::Bytes;
pub use endpoint::Endpoint;
@@ -385,18 +376,15 @@ pub mod metrics {
pub mod notification {
pub use crate::services::notification_sys::{
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
start_remote_version_state_fleet_probe,
};
}
pub mod object {
pub use crate::object_api::{
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo,
ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode,
ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
unregister_object_mutation_hook,
BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer,
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
};
pub use crate::store::PreparedGetObjectReader;
}
@@ -418,15 +406,13 @@ pub mod rio {
pub mod rpc {
pub use crate::cluster::rpc::{
AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature,
};
}
@@ -8974,7 +8974,7 @@ mod tests {
.await
.expect("first worker result should persist");
assert_eq!(first.state, ManualTransitionJobState::Running);
assert_eq!(first.report.transition_completed, 0);
assert_eq!(first.report.transition_completed, 1);
assert_eq!(first.report.transition_failed, 0);
let duplicate = record_manual_transition_worker_result(
@@ -8987,11 +8987,11 @@ mod tests {
.await
.expect("duplicate worker result should be idempotent");
assert_eq!(duplicate.state, ManualTransitionJobState::Running);
assert_eq!(duplicate.report.transition_completed, 0);
assert_eq!(duplicate.report.transition_completed, 1);
assert_eq!(duplicate.report.transition_failed, 0);
let second_key = manual_transition_worker_result_task_key(&bucket, "logs/b", None);
let pending_record = record_manual_transition_worker_result(
let final_record = record_manual_transition_worker_result(
ecstore.clone(),
job_id,
&second_key,
@@ -9000,14 +9000,7 @@ mod tests {
)
.await
.expect("second distinct worker result should persist");
assert_eq!(pending_record.state, ManualTransitionJobState::Running);
assert_eq!(pending_record.report.transition_completed, 0);
assert_eq!(pending_record.report.transition_failed, 0);
let final_record =
reconcile_manual_transition_worker_results(ecstore.clone(), job_id, ManualTransitionQueueSnapshot::default())
.await
.expect("worker result journal should reconcile");
assert_eq!(final_record.state, ManualTransitionJobState::Partial);
assert_eq!(final_record.report.transition_completed, 1);
assert_eq!(final_record.report.transition_failed, 1);
@@ -9042,7 +9035,7 @@ mod tests {
.expect("worker result job record should save");
let task_key = manual_transition_worker_result_task_key(&bucket, "logs/fail", None);
let pending_record = record_manual_transition_worker_result_with_reason(
let final_record = record_manual_transition_worker_result_with_reason(
ecstore.clone(),
job_id,
&task_key,
@@ -9052,12 +9045,7 @@ mod tests {
)
.await
.expect("worker result with failure reason should persist");
assert!(pending_record.report.tier_failure_by_reason.is_empty());
assert_eq!(pending_record.report.transition_failed, 0);
let final_record = reconcile_manual_transition_worker_results(ecstore, job_id, ManualTransitionQueueSnapshot::default())
.await
.expect("worker failure reason should reconcile");
assert_eq!(
final_record
.report
@@ -11318,25 +11306,23 @@ mod tests {
// Distinct payloads with distinct sizes: a mixed-generation reassembly
// would produce bytes matching none of them (or fail the read outright).
let candidates: Vec<Vec<u8>> = (0..2)
let candidates: Vec<Vec<u8>> = (0..3)
.map(|g| {
let len = 4096 + g * 512;
vec![b'a' + g as u8; len]
})
.collect();
let commit_barrier = MultipartCommitBarrier::install_for_arrivals(
&bucket,
object,
MultipartCommitPause::PutPartBeforeLockAcquire,
candidates.len(),
);
let commit_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
let start = Arc::new(tokio::sync::Barrier::new(candidates.len() + 1));
let mut tasks = tokio::task::JoinSet::new();
for payload in candidates.iter().cloned() {
let store = ecstore.clone();
let bucket = bucket.clone();
let upload_id = upload.upload_id.clone();
let start = Arc::clone(&start);
tasks.spawn(async move {
start.wait().await;
let mut data = PutObjReader::from_vec(payload.clone());
store
.put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
@@ -11344,10 +11330,11 @@ mod tests {
.map(|info| (info, payload))
});
}
start.wait().await;
// Both writers finish streaming before racing for the uploadId commit
// lock. Two generations are sufficient to exercise the mixed-shard
// hazard, while each waiter sits behind at most one cross-disk rename.
// The first writer holds the uploadId commit lock while the other
// resends reach the same critical section. Releasing it proves the
// handoff without depending on saturated CI disk latency.
commit_barrier.wait_until_paused().await;
commit_barrier.release();
@@ -1646,14 +1646,40 @@ pub async fn record_manual_transition_worker_result_with_reason(
job_id: Uuid,
task_key: &str,
result: ManualTransitionWorkerResult,
_queue_snapshot: ManualTransitionQueueSnapshot,
queue_snapshot: ManualTransitionQueueSnapshot,
failure_reason: Option<ManualTransitionWorkerFailureReason>,
) -> EcstoreResult<ManualTransitionJobRecord> {
let result_record = ManualTransitionWorkerResultRecord::new_with_reason(job_id, task_key, result, failure_reason);
if !save_manual_transition_worker_result_if_absent(api.clone(), &result_record).await? {
return load_manual_transition_job_record(api, job_id).await;
}
load_manual_transition_job_record(api, job_id).await
for _ in 0..4 {
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
if record.is_terminal() {
return Ok(record);
}
record.record_worker_result_with_reason(result, queue_snapshot, failure_reason);
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
Ok(()) => {
if record.is_terminal() {
delete_manual_transition_scope_admission_if_current(
api.clone(),
&record.scope_key,
record.job_id,
record.lease_id,
)
.await?;
} else {
renew_manual_transition_scope_admission_from_job(api, &record).await?;
}
return Ok(record);
}
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
}
Err(Error::PreconditionFailed)
}
pub async fn renew_manual_transition_job_lease(
@@ -23,7 +23,6 @@ use uuid::Uuid;
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::bucket::lifecycle::tier_sweeper::{
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
};
@@ -617,199 +616,6 @@ pub enum TransitionTransactionRecoveryOutcome {
Retained,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TransitionOperatorProbe {
Missing,
UnversionedPresent,
VersionedPresent(String),
Ambiguous,
Unsupported,
}
impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
fn from(value: TransitionCandidateProbe) -> Self {
match value {
TransitionCandidateProbe::Missing => Self::Missing,
TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
TransitionCandidateProbe::Unsupported => Self::Unsupported,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct TransitionOperatorStatus {
pub transaction_id: Uuid,
pub state: TransitionTransactionState,
pub tier_name: String,
pub remote_object: String,
pub not_after_unix_nanos: i64,
pub probe: TransitionOperatorProbe,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct TransitionOperatorDeleteResult {
pub status: TransitionOperatorStatus,
pub journal_observed_after_delete: bool,
}
#[derive(Debug, thiserror::Error)]
pub enum TransitionOperatorError {
#[error("transition transaction was not found")]
NotFound,
#[error("transition transaction is still inside its active ownership window")]
NotExpired,
#[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
InvalidState(TransitionTransactionState),
#[error("an exact non-empty remote version is required")]
RemoteVersionRequired,
#[error("remote candidate is not proven missing: {0:?}")]
CandidateNotMissing(TransitionOperatorProbe),
#[error("remote candidate version does not match requested exact version: expected {expected}, observed {actual:?}")]
CandidateVersionMismatch {
expected: String,
actual: TransitionOperatorProbe,
},
#[error("transition transaction store failed: {0}")]
Store(#[source] Error),
#[error("remote tier reconciliation failed: {0}")]
Remote(#[source] std::io::Error),
}
type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
fn validate_operator_reconcile_transaction(
transaction: &TransitionTransaction,
now_unix_nanos: i128,
) -> TransitionOperatorResult<()> {
transaction
.validate()
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
return Err(TransitionOperatorError::InvalidState(transaction.state));
}
if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
return Err(TransitionOperatorError::NotExpired);
}
Ok(())
}
async fn load_operator_reconcile_transaction(
api: Arc<ECStore>,
transaction_id: Uuid,
) -> TransitionOperatorResult<TransitionTransaction> {
match load_transition_transaction_record(api, transaction_id).await {
Ok(transaction) => Ok(transaction),
Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
Err(err) => Err(TransitionOperatorError::Store(err)),
}
}
async fn operator_probe_transition_candidate(
api: Arc<ECStore>,
transaction: &TransitionTransaction,
) -> TransitionOperatorResult<TransitionOperatorProbe> {
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
&api.tier_config_mgr(),
&transaction.tier_name,
transaction.backend_fingerprint,
)
.await
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
lease
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
.await
.map(TransitionOperatorProbe::from)
.map_err(TransitionOperatorError::Remote)
}
pub async fn inspect_transition_transaction_for_operator(
api: Arc<ECStore>,
transaction_id: Uuid,
) -> TransitionOperatorResult<TransitionOperatorStatus> {
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
let probe = operator_probe_transition_candidate(api, &transaction).await?;
Ok(TransitionOperatorStatus {
transaction_id,
state: transaction.state,
tier_name: transaction.tier_name,
remote_object: transaction.remote_object,
not_after_unix_nanos: transaction.not_after_unix_nanos,
probe,
})
}
pub async fn delete_transition_candidate_for_operator(
api: Arc<ECStore>,
transaction_id: Uuid,
remote_version_id: &str,
) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
if remote_version_id.is_empty() {
return Err(TransitionOperatorError::RemoteVersionRequired);
}
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
&api.tier_config_mgr(),
&transaction.tier_name,
transaction.backend_fingerprint,
)
.await
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
lease
.validate_remote_version_id(remote_version_id)
.map_err(TransitionOperatorError::Remote)?;
let before_delete_probe = lease
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
.await
.map(TransitionOperatorProbe::from)
.map_err(TransitionOperatorError::Remote)?;
if !matches!(&before_delete_probe, TransitionOperatorProbe::VersionedPresent(version_id) if version_id == remote_version_id) {
return Err(TransitionOperatorError::CandidateVersionMismatch {
expected: remote_version_id.to_string(),
actual: before_delete_probe,
});
}
delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
.await
.map_err(TransitionOperatorError::Remote)?;
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
Ok(_) => true,
Err(Error::ConfigNotFound) => false,
Err(err) => return Err(TransitionOperatorError::Store(err)),
};
Ok(TransitionOperatorDeleteResult {
status: TransitionOperatorStatus {
transaction_id,
state: transaction.state,
tier_name: transaction.tier_name,
remote_object: transaction.remote_object,
not_after_unix_nanos: transaction.not_after_unix_nanos,
probe,
},
journal_observed_after_delete,
})
}
pub async fn finalize_missing_transition_transaction_for_operator(
api: Arc<ECStore>,
transaction_id: Uuid,
) -> TransitionOperatorResult<()> {
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
if probe != TransitionOperatorProbe::Missing {
return Err(TransitionOperatorError::CandidateNotMissing(probe));
}
delete_transition_transaction_record(api, transaction_id)
.await
.map_err(TransitionOperatorError::Store)
}
pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
let transaction_id = transition_transaction_id_from_record_object_name(object)?;
TransitionTransaction::decode(transaction_id, data)
@@ -894,7 +700,7 @@ async fn recover_unknown_upload_outcome(
.map_err(Error::other)?;
match lease
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
.probe_transition_candidate(&transaction.remote_object)
.await
.map_err(Error::other)?
{
@@ -1291,27 +1097,6 @@ mod tests {
.expect("upload state change should succeed")
}
#[test]
fn operator_reconcile_requires_expired_unknown_upload_outcome() {
let mut transaction = new_transaction();
let active_deadline = transaction.not_after_unix_nanos;
assert!(matches!(
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
));
transaction
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
.expect("unknown upload outcome should be recorded");
assert!(matches!(
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
Err(TransitionOperatorError::NotExpired)
));
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
.expect("expired unknown upload outcome should be eligible");
}
fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
TransitionCleanupProof {
transaction_id: transaction.transaction_id,
File diff suppressed because it is too large Load Diff
@@ -172,7 +172,7 @@ impl ProviderVersionCapabilities {
}
}
pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
if version_id.is_empty() {
return Err(Error::new(
ErrorKind::InvalidData,
+15 -1
View File
@@ -46,6 +46,16 @@ lazy_static! {
m.insert("x-amz-replication-status".to_string(), true);
m
};
static ref SSE_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("x-amz-server-side-encryption".to_string(), true);
m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
m.insert("x-amz-server-side-encryption-context".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
m
};
}
pub fn is_standard_query_value(qs_key: &str) -> bool {
@@ -60,12 +70,16 @@ pub fn is_standard_header(header_key: &str) -> bool {
*SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_sse_header(header_key: &str) -> bool {
*SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_amz_header(header_key: &str) -> bool {
let key = header_key.to_lowercase();
key.starts_with("x-amz-meta-")
|| key.starts_with("x-amz-grant-")
|| key == "x-amz-acl"
|| rustfs_utils::http::is_sse_header(header_key)
|| is_sse_header(header_key)
|| key.starts_with("x-amz-checksum-")
}
+9 -214
View File
@@ -12,20 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
use crate::cluster::rpc::http_auth::{
RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
};
use crate::cluster::rpc::{
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
};
#[cfg(test)]
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
use crate::runtime::sources as runtime_sources;
use http::{Request as HttpRequest, Response as HttpResponse, Uri};
use http::Uri;
use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class,
proto_gen::node_service::{
@@ -33,19 +24,9 @@ use rustfs_protos::{
tier_mutation_control_service_client::TierMutationControlServiceClient,
},
};
use std::{
collections::HashMap,
error::Error,
future::Future,
io::ErrorKind,
pin::Pin,
sync::{LazyLock, Mutex},
task::{Context, Poll},
};
use std::{error::Error, io::ErrorKind};
use tonic::{service::interceptor::InterceptedService, transport::Channel};
use tower::Service;
use tracing::debug;
use uuid::Uuid;
use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
@@ -54,7 +35,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c
pub async fn node_service_time_out_client(
addr: &String,
interceptor: TonicInterceptor,
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
// Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
// `_for_class` variant below (grpc-optimization P1).
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
@@ -63,14 +44,13 @@ pub async fn node_service_time_out_client(
pub async fn heal_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr).await?,
};
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
@@ -79,14 +59,13 @@ pub async fn heal_control_time_out_client(
pub async fn tier_mutation_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr).await?,
};
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
@@ -102,7 +81,7 @@ pub async fn node_service_time_out_client_for_class(
addr: &String,
interceptor: TonicInterceptor,
class: ChannelClass,
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match class {
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
@@ -117,7 +96,6 @@ pub async fn node_service_time_out_client_for_class(
};
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
@@ -125,7 +103,7 @@ pub async fn node_service_time_out_client_for_class(
pub async fn node_service_time_out_client_no_auth(
addr: &String,
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
}
@@ -221,104 +199,6 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
}
}
/// The transport service that learns an authenticated peer boot epoch and adds the replay-scoped
/// signature only after one has been observed. The v1/v2 interceptor stays inside this wrapper so
/// old servers continue receiving precisely the metadata they understand.
#[derive(Clone, Debug)]
pub struct ReplayScopeChannel<S> {
inner: S,
audience: Option<String>,
}
/// The channel type used by internode clients after v2 authentication and replay-scope handling.
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
impl<S> ReplayScopeChannel<S> {
fn new(inner: S, audience: Option<String>) -> Self {
Self { inner, audience }
}
}
fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
}
fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
epochs.insert(audience, epoch);
}
}
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
where
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
S::Error: Send + 'static,
S::Future: Send + 'static,
ReqBody: Send + 'static,
ResBody: Send + 'static,
{
type Response = HttpResponse<ResBody>;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
let authenticated = self.audience.as_ref().is_some_and(|_| {
request
.headers()
.get(RPC_AUTH_VERSION_HEADER)
.and_then(|value| value.to_str().ok())
== Some(RPC_AUTH_VERSION_V2)
});
let challenge = authenticated.then(Uuid::new_v4);
if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
// The challenge is independently HMAC-authenticated by the response proof. It is not
// part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
request.headers_mut().insert(
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
challenge.to_string().parse().expect("UUID must be a valid header value"),
);
if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
cached_peer_boot_epoch(audience),
request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
request
.headers()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok()),
) {
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
Ok(headers) => request.headers_mut().extend(headers),
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
}
}
}
let audience = self.audience.clone();
let future = self.inner.call(request);
Box::pin(async move {
let response = future.await?;
if let (Some(audience), Some(challenge)) = (audience, challenge) {
match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
Err(error)
if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
|| response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
{
debug!(error = %error, "peer boot epoch response proof was rejected")
}
Err(_) => {}
}
}
Ok(response)
})
}
}
pub struct TonicSignatureInterceptor {
audience: Option<String>,
}
@@ -377,13 +257,6 @@ impl TonicInterceptor {
}
Ok(self)
}
fn replay_scope_audience(&self) -> Option<String> {
match self {
Self::Signature(interceptor) => interceptor.audience.clone(),
Self::NoOp(_) => None,
}
}
}
impl tonic::service::Interceptor for TonicInterceptor {
@@ -406,38 +279,6 @@ mod tests {
use tracing_opentelemetry::OpenTelemetrySpanExt;
use tracing_subscriber::{Registry, layer::SubscriberExt};
#[derive(Clone)]
struct EpochProofService {
audience: String,
seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
}
impl Service<HttpRequest<()>> for EpochProofService {
type Response = HttpResponse<()>;
type Error = std::convert::Infallible;
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
self.seen_headers
.lock()
.expect("test header capture lock must not be poisoned")
.push(request.headers().clone());
let challenge = tonic_boot_epoch_challenge(request.headers())
.expect("client challenge must be syntactically valid")
.expect("authenticated client request must carry a boot epoch challenge");
let mut response = HttpResponse::new(());
response.headers_mut().extend(
tonic_boot_epoch_response_headers(&self.audience, challenge)
.expect("test server must be able to sign an epoch proof"),
);
std::future::ready(Ok(response))
}
}
fn ensure_test_rpc_secret() {
runtime_sources::ensure_test_rpc_secret();
}
@@ -579,52 +420,6 @@ mod tests {
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
}
#[test]
fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
ensure_test_rpc_secret();
let audience = "replay-scope-client-test:9000";
PEER_BOOT_EPOCHS
.lock()
.expect("peer epoch cache lock must not be poisoned")
.remove(audience);
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
let service = EpochProofService {
audience: audience.to_string(),
seen_headers: seen_headers.clone(),
};
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
let make_request = || {
let mut request = HttpRequest::builder()
.uri("/node_service.NodeService/Ping")
.body(())
.expect("test RPC request must build");
request.headers_mut().extend(
gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
.expect("v2 test headers must mint"),
);
request
};
futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
assert_eq!(headers.len(), 2);
assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
assert!(
!headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
"the first request must remain v2-compatible until the peer proves its epoch"
);
assert!(
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
"the second request must carry the replay-scoped v3 signature"
);
PEER_BOOT_EPOCHS
.lock()
.expect("peer epoch cache lock must not be poisoned")
.remove(audience);
}
#[test]
fn test_signature_interceptor_requires_generated_method_metadata() {
ensure_test_rpc_secret();
+11 -530
View File
@@ -20,8 +20,8 @@
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
//! below, plus the broader negative-signature suite. The advisory class is: a
//! node must never accept an RPC whose auth is missing, malformed, or signed
//! with the default/empty shared secret. Body-bound v2 requests and all replay-scoped v3
//! requests additionally receive process-local replay protection. See
//! with the default/empty shared secret. Body-bound v2 requests additionally
//! receive process-local replay protection. See
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
//!
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
@@ -50,22 +50,13 @@ use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
pub(crate) const RPC_AUTH_VERSION_V2: &str = "2";
pub const RPC_REPLAY_SCOPE_VERSION_HEADER: &str = "x-rustfs-rpc-replay-scope-version";
pub const RPC_REPLAY_SCOPE_SIGNATURE_HEADER: &str = "x-rustfs-rpc-signature-v3";
pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
const RPC_AUTH_VERSION_V2: &str = "2";
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
@@ -84,15 +75,9 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
)
});
static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
get_env_bool(
rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
)
});
// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
// cannot disable replay protection by rejecting every body-bound request.
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
rustfs_utils::get_env_usize(
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
@@ -101,7 +86,6 @@ static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
.max(1)
});
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
#[derive(Default)]
struct RpcNonceCache {
@@ -329,189 +313,6 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
mac.verify_slice(&signature).is_ok()
}
#[derive(Clone, Copy)]
struct ReplayScope<'a> {
audience: &'a str,
path: &'a str,
timestamp: &'a str,
nonce: Uuid,
content_sha256: &'a str,
boot_epoch: Uuid,
}
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
for part in [
scope.audience.as_bytes(),
b"|",
scope.path.as_bytes(),
b"|POST|",
scope.timestamp.as_bytes(),
b"|",
scope.nonce.as_bytes(),
b"|",
scope.content_sha256.as_bytes(),
b"|",
scope.boot_epoch.as_bytes(),
] {
mac.update(part);
}
}
fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std::io::Result<String> {
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_replay_scope(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
}
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
return false;
};
update_replay_scope(&mut mac, scope);
mac.verify_slice(&signature).is_ok()
}
fn update_boot_epoch_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
mac.update(RPC_BOOT_EPOCH_PROOF_DOMAIN);
mac.update(audience.as_bytes());
mac.update(b"|");
mac.update(challenge.as_bytes());
mac.update(boot_epoch.as_bytes());
}
fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid) -> std::io::Result<String> {
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
}
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
}
let proof = general_purpose::STANDARD
.decode(proof)
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
mac.verify_slice(&proof)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
}
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
(!value.is_nil())
.then_some(value)
.ok_or_else(|| std::io::Error::other(format!("Invalid {name}")))
}
fn parse_tonic_rpc_path(path: &str) -> std::io::Result<(&str, &str)> {
path.strip_prefix('/')
.and_then(|path| path.split_once('/'))
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))
}
/// The process-unique epoch included in every replay-scoped server verification.
///
/// A fresh process gets a fresh value, so a signature captured before a server restart cannot be
/// admitted even though the bounded in-memory nonce cache necessarily starts empty again.
pub fn tonic_rpc_boot_epoch() -> Uuid {
*RPC_BOOT_EPOCH
}
/// Build the additive replay-scope headers for a request that already carries rolling-upgrade-safe
/// v1/v2 metadata. `timestamp` and `content_sha256` are deliberately reused from the v2 scope so
/// old servers can continue validating the same request unchanged.
pub fn gen_tonic_replay_scope_headers(
audience: &str,
path: &str,
timestamp: &str,
content_sha256: &str,
boot_epoch: Uuid,
) -> std::io::Result<HeaderMap> {
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
}
parse_tonic_rpc_path(path)?;
timestamp
.parse::<i64>()
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
let nonce = Uuid::new_v4();
let signature = generate_replay_scope_signature(
&get_shared_secret()?,
ReplayScope {
audience,
path,
timestamp,
nonce,
content_sha256,
boot_epoch,
},
)?;
let mut headers = HeaderMap::new();
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
headers.insert(
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
header_value(&signature, RPC_REPLAY_SCOPE_SIGNATURE_HEADER)?,
);
headers.insert(
RPC_REPLAY_SCOPE_NONCE_HEADER,
header_value(&nonce.to_string(), RPC_REPLAY_SCOPE_NONCE_HEADER)?,
);
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
Ok(headers)
}
/// Parse the optional client challenge used to authenticate a server boot-epoch advertisement.
pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option<Uuid>> {
headers
.get(RPC_BOOT_EPOCH_CHALLENGE_HEADER)
.map(|value| {
value
.to_str()
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch challenge"))
.and_then(|value| non_nil_uuid(value, "RPC boot epoch challenge"))
})
.transpose()
}
/// Build the authenticated response headers for a client boot-epoch challenge.
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
let boot_epoch = tonic_rpc_boot_epoch();
let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
let mut headers = HeaderMap::new();
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
Ok(headers)
}
/// Verify the server boot-epoch response for a challenge generated by this client.
pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
let boot_epoch = headers
.get(RPC_BOOT_EPOCH_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
let proof = headers
.get(RPC_BOOT_EPOCH_PROOF_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
Ok(boot_epoch)
}
fn valid_content_sha256(value: &str) -> bool {
value == UNSIGNED_PAYLOAD
|| (value.len() == 64
@@ -642,16 +443,6 @@ pub fn set_tonic_canonical_body_digest<T>(request: &mut tonic::Request<T>, canon
Ok(())
}
pub fn set_tonic_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(
request: &mut tonic::Request<T>,
) -> std::io::Result<()> {
let canonical_body = request
.get_ref()
.canonical_body()
.map_err(|_| std::io::Error::other("RPC mutation body length cannot be represented"))?;
set_tonic_canonical_body_digest(request, &canonical_body)
}
pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
let version = request
.metadata()
@@ -675,7 +466,7 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
Ok(())
}
/// Verify a mutating RPC's canonical body digest with a rolling-upgrade fallback.
/// Verify a mutating disk RPC's canonical body digest with a rolling-upgrade fallback.
///
/// When the request carries a real (non-`UNSIGNED-PAYLOAD`) content SHA-256 it is verified exactly
/// like [`verify_tonic_canonical_body_digest`]. The digest value is a member of the signed v2
@@ -706,7 +497,7 @@ fn verify_tonic_mutation_body_digest_with_strictness<T>(
Some(digest) if digest != UNSIGNED_PAYLOAD => verify_tonic_canonical_body_digest(request, canonical_body),
_ => {
// RUSTFS_COMPAT_TODO(disk-mutation-body-digest): accept digestless peers during rolling upgrades. Remove after the
// minimum supported RustFS peer version body-binds every mutating RPC.
// minimum supported RustFS peer version body-binds every mutating disk RPC.
if strict {
return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
}
@@ -730,17 +521,6 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
.any(|name| headers.contains_key(*name))
}
fn has_replay_scope_headers(headers: &HeaderMap) -> bool {
[
RPC_REPLAY_SCOPE_VERSION_HEADER,
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
RPC_REPLAY_SCOPE_NONCE_HEADER,
RPC_BOOT_EPOCH_HEADER,
]
.iter()
.any(|name| headers.contains_key(*name))
}
/// Whether the server requires target-bound v2 authentication on every internode gRPC request,
/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
/// lever gated on the v1-fallback counter reading zero fleet-wide; see
@@ -750,127 +530,9 @@ fn internode_rpc_signature_strict() -> bool {
*INTERNODE_RPC_SIGNATURE_STRICT
}
fn internode_rpc_replay_scope_strict() -> bool {
*INTERNODE_RPC_REPLAY_SCOPE_STRICT
}
fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
if audience.is_empty() {
return Err(std::io::Error::other("Missing RPC audience"));
}
parse_tonic_rpc_path(path)?;
let version = headers
.get(RPC_REPLAY_SCOPE_VERSION_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope version"))?;
if version != RPC_REPLAY_SCOPE_VERSION_V3 {
return Err(std::io::Error::other("Unsupported RPC replay scope version"));
}
let signature = headers
.get(RPC_REPLAY_SCOPE_SIGNATURE_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope signature"))?;
let timestamp = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
let signed_at = timestamp
.parse::<i64>()
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
check_timestamp(signed_at)?;
let nonce = headers
.get(RPC_REPLAY_SCOPE_NONCE_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope nonce"))
.and_then(|value| non_nil_uuid(value, "RPC replay scope nonce"))?;
let content_sha256 = headers
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
if !valid_content_sha256(content_sha256) {
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
}
let boot_epoch = headers
.get(RPC_BOOT_EPOCH_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
let secret = get_shared_secret()?;
if !verify_replay_scope_signature(
&secret,
ReplayScope {
audience,
path,
timestamp,
nonce,
content_sha256,
boot_epoch,
},
signature,
) {
return Err(std::io::Error::other("Invalid RPC replay scope signature"));
}
if boot_epoch != tonic_rpc_boot_epoch() {
return Err(std::io::Error::other("RPC boot epoch is stale"));
}
check_and_record_nonce(nonce, signed_at)
}
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
verify_tonic_rpc_signature_with_policy(
audience,
path,
headers,
internode_rpc_signature_strict(),
internode_rpc_replay_scope_strict(),
false,
)
}
/// Verify gRPC authentication while allowing the narrowly scoped v2 `Ping` bootstrap used to
/// obtain an authenticated server boot epoch when replay-scope strictness is enabled.
pub fn verify_tonic_rpc_signature_with_bootstrap(
audience: &str,
path: &str,
headers: &HeaderMap,
allow_replay_scope_bootstrap: bool,
) -> std::io::Result<()> {
verify_tonic_rpc_signature_with_policy(
audience,
path,
headers,
internode_rpc_signature_strict(),
internode_rpc_replay_scope_strict(),
allow_replay_scope_bootstrap,
)
}
fn verify_tonic_rpc_signature_with_policy(
audience: &str,
path: &str,
headers: &HeaderMap,
signature_strict: bool,
replay_scope_strict: bool,
allow_replay_scope_bootstrap: bool,
) -> std::io::Result<()> {
if has_replay_scope_headers(headers) {
return verify_tonic_replay_scope_signature(audience, path, headers);
}
// Only a method-bound v2 Ping with a syntactically valid challenge may bootstrap a strict
// client after its peer restarts. Legacy metadata never gets this exception.
let bootstrap = allow_replay_scope_bootstrap
&& has_v2_auth_headers(headers)
&& tonic_boot_epoch_challenge(headers).is_ok_and(|challenge| challenge.is_some());
if replay_scope_strict && !bootstrap {
return Err(std::io::Error::other("RPC replay-scoped authentication required"));
}
verify_tonic_rpc_signature_with_strictness(audience, path, headers, signature_strict)?;
global_internode_metrics().record_replay_scope_fallback();
Ok(())
verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
}
/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
@@ -1015,28 +677,11 @@ mod tests {
use crate::cluster::rpc::context_propagation::REQUEST_ID_HEADER;
use crate::runtime::sources as runtime_sources;
use http::{HeaderMap, Method};
use rustfs_protos::{
CanonicalMutationBody as _, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
proto_gen::node_service::{Mss, SignalServiceRequest},
};
use std::collections::HashMap;
use std::io::{self, Write};
use std::sync::{Arc, Mutex};
use time::OffsetDateTime;
use tracing_subscriber::fmt::MakeWriter;
fn signal_service_request(signal: &str, sub_system: &str, dry_run: &str) -> SignalServiceRequest {
SignalServiceRequest {
vars: Some(Mss {
value: HashMap::from([
(PEER_RESTSIGNAL.to_string(), signal.to_string()),
(PEER_RESTSUB_SYS.to_string(), sub_system.to_string()),
(PEER_RESTDRY_RUN.to_string(), dry_run.to_string()),
]),
}),
}
}
#[derive(Clone, Default)]
struct CapturedLogs {
buffer: Arc<Mutex<Vec<u8>>>,
@@ -1601,104 +1246,6 @@ mod tests {
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
}
#[test]
fn replay_scope_binds_path_epoch_and_random_nonce() {
ensure_test_rpc_secret();
let path = "/node_service.NodeService/Ping";
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("v2 compatibility headers should build");
let timestamp = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 timestamp")
.to_string();
let content_sha256 = headers
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 content digest")
.to_string();
headers.extend(
gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, tonic_rpc_boot_epoch())
.expect("replay-scope headers should build"),
);
assert!(
verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false).is_ok(),
"the first replay-scoped request must be accepted"
);
let replay = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false)
.expect_err("the random replay-scope nonce must be single-use");
assert_eq!(replay.to_string(), "RPC request replay detected");
let path_error = verify_tonic_replay_scope_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
.expect_err("a replay-scoped signature must not move to another method");
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
}
#[test]
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
ensure_test_rpc_secret();
let path = "/node_service.NodeService/Ping";
let mut partial = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("v2 compatibility headers should build");
partial.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
let error = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
.expect_err("partial replay-scope metadata must never downgrade to v2");
assert_eq!(error.to_string(), "Missing RPC replay scope signature");
let timestamp = partial
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 timestamp")
.to_string();
let content_sha256 = partial
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 content digest")
.to_string();
let stale_epoch = Uuid::new_v4();
partial.extend(
gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, stale_epoch)
.expect("replay-scope headers should build"),
);
let stale = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
.expect_err("a signature from a prior server boot epoch must be rejected");
assert_eq!(stale.to_string(), "RPC boot epoch is stale");
}
#[test]
fn replay_scope_strictness_allows_only_authenticated_ping_bootstrap() {
ensure_test_rpc_secret();
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("v2 compatibility headers should build");
let rejected =
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, false)
.expect_err("strict replay scope must reject stripped new metadata");
assert_eq!(rejected.to_string(), "RPC replay-scoped authentication required");
headers.insert(
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
HeaderValue::from_str(&Uuid::new_v4().to_string()).expect("UUID header"),
);
assert!(
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, true,)
.is_ok(),
"only the signed Ping bootstrap may obtain a new server epoch in strict mode"
);
}
#[test]
fn boot_epoch_response_proof_binds_audience_challenge_and_epoch() {
ensure_test_rpc_secret();
let challenge = Uuid::new_v4();
let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("proof headers should build");
let epoch =
verify_tonic_boot_epoch_response("node-a:9000", challenge, &headers).expect("matching proof headers should verify");
assert_eq!(epoch, tonic_rpc_boot_epoch());
assert!(verify_tonic_boot_epoch_response("node-b:9000", challenge, &headers).is_err());
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
}
#[test]
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
ensure_test_rpc_secret();
@@ -2049,72 +1596,6 @@ mod tests {
assert_eq!(stripped.to_string(), "RPC content SHA-256 mismatch");
}
#[test]
fn signal_service_mutation_contract_rejects_tampering_and_replay() {
ensure_test_rpc_secret();
let body = signal_service_request("2", "scanner", "false")
.canonical_body()
.expect("small signal request should encode");
let mut request = tonic::Request::new(());
set_tonic_canonical_body_digest(&mut request, &body).expect("canonical body digest should be attached");
let content_sha256 = request
.metadata()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok());
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "SignalService", content_sha256)
.expect("body-bound auth headers should build");
request.metadata_mut().as_mut().extend(headers.clone());
assert!(
verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers).is_ok(),
"the first body-bound signal request must authenticate"
);
assert!(verify_tonic_mutation_body_digest(&request, &body).is_ok());
let tampered = signal_service_request("1", "scanner", "false")
.canonical_body()
.expect("small signal request should encode");
let error = verify_tonic_mutation_body_digest(&request, &tampered)
.expect_err("changing the signal must invalidate the signed digest");
assert_eq!(error.to_string(), "RPC content SHA-256 mismatch");
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
.expect_err("reusing the signal nonce must fail");
assert_eq!(replay.to_string(), "RPC request replay detected");
}
#[test]
#[serial_test::serial(rpc_body_digest_fallback_counter)]
fn signal_service_mutation_contract_preserves_rollout_fallback_and_strictness() {
let body = signal_service_request("2", "scanner", "false")
.canonical_body()
.expect("small signal request should encode");
let before = global_internode_metrics().snapshot().body_digest_fallback_total;
let digestless = tonic::Request::new(());
assert!(
verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, false).is_ok(),
"old peers must remain compatible while the rollout gate is open"
);
assert_eq!(
global_internode_metrics().snapshot().body_digest_fallback_total,
before + 1,
"accepted digestless signal requests must be visible in the fallback metric"
);
let error = verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, true)
.expect_err("strict mode must reject a digestless signal request");
assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
let mut bound = tonic::Request::new(());
set_tonic_canonical_body_digest(&mut bound, &body).expect("canonical body digest should be attached");
bound
.metadata_mut()
.as_mut()
.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
assert!(verify_tonic_mutation_body_digest_with_strictness(&bound, &body, true).is_ok());
}
#[test]
fn nonce_cache_rejects_replay_after_wall_clock_regression() {
let now = Instant::now();
+5 -10
View File
@@ -26,18 +26,13 @@ pub(crate) mod runtime_sources;
pub use background_monitor::shutdown_background_monitors;
pub(crate) use background_monitor::spawn_background_monitor;
pub use client::{
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth,
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
};
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
#[allow(unused_imports)]
pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
@@ -13,10 +13,10 @@
// limitations under the License.
use crate::cluster::rpc::client::{
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
};
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof};
use crate::error::{Error, Result};
use crate::storage_api_contracts::internode::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
@@ -50,7 +50,6 @@ use rustfs_protos::proto_gen::node_service::{
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
};
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
use rustfs_utils::XHost;
use serde::{Deserialize, Serialize as _};
@@ -66,9 +65,13 @@ use std::{
use tokio::{net::TcpStream, time::Duration};
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::{debug, info, warn};
use uuid::Uuid;
pub const PEER_RESTSIGNAL: &str = "signal";
pub const PEER_RESTSUB_SYS: &str = "sub-sys";
pub const PEER_RESTDRY_RUN: &str = "dry-run";
pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
@@ -221,21 +224,6 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
}
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
if topology_member != expected_member {
return Err(Error::other(
"peer returned a remote version state capability for a different topology member",
));
}
let server_epoch =
Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
if server_epoch.is_nil() {
return Err(Error::other("peer returned a nil remote version state epoch"));
}
Ok(server_epoch)
}
#[derive(Clone, Debug)]
pub struct PeerLiveEventsBatch {
pub events: Vec<u8>,
@@ -247,7 +235,6 @@ pub struct PeerLiveEventsBatch {
pub struct PeerRestClient {
pub host: XHost,
pub grid_host: String,
topology_member: String,
offline: Arc<AtomicBool>,
recovery_running: Arc<AtomicBool>,
}
@@ -340,11 +327,9 @@ impl PeerRestClient {
}
pub fn new(host: XHost, grid_host: String) -> Self {
let topology_member = host.to_string();
Self {
host,
grid_host,
topology_member,
offline: Arc::new(AtomicBool::new(false)),
recovery_running: Arc::new(AtomicBool::new(false)),
}
@@ -364,11 +349,7 @@ impl PeerRestClient {
let client = match grid_host {
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
Ok(host) => {
let mut client = PeerRestClient::new(host, grid_host);
client.topology_member = peer_host_port.clone();
Some(client)
}
Ok(host) => Some(PeerRestClient::new(host, grid_host)),
Err(err) => {
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
None
@@ -411,7 +392,7 @@ impl PeerRestClient {
(remote, all, remote_topology_hosts)
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
if self.offline.load(Ordering::Acquire) {
self.mark_offline_and_spawn_recovery();
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -432,7 +413,7 @@ impl PeerRestClient {
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
InterceptedService<Channel, TonicInterceptor>,
>,
> {
if self.offline.load(Ordering::Acquire) {
@@ -453,7 +434,7 @@ impl PeerRestClient {
async fn get_tier_mutation_control_client(
&self,
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
if self.offline.load(Ordering::Acquire) {
self.mark_offline_and_spawn_recovery();
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -1175,24 +1156,14 @@ impl PeerRestClient {
validate_heal_control_capability_proof(&canonical_ack, &proof)
}
pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
let result = self
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
.await?;
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
Ok((self.topology_member.clone(), epoch))
}
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadBucketMetadataRequest {
let request = Request::new(LoadBucketMetadataRequest {
bucket: bucket.to_string(),
scanner_maintenance_change,
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.load_bucket_metadata(request).await?.into_inner();
if !response.success {
@@ -1212,10 +1183,9 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(DeleteBucketMetadataRequest {
let request = Request::new(DeleteBucketMetadataRequest {
bucket: bucket.to_string(),
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.delete_bucket_metadata(request).await?.into_inner();
if !response.success {
@@ -1235,10 +1205,9 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(DeletePolicyRequest {
let request = Request::new(DeletePolicyRequest {
policy_name: policy.to_string(),
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.delete_policy(request).await?.into_inner();
if !response.success {
@@ -1258,10 +1227,9 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadPolicyRequest {
let request = Request::new(LoadPolicyRequest {
policy_name: policy.to_string(),
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.load_policy(request).await?.into_inner();
if !response.success {
@@ -1281,12 +1249,11 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadPolicyMappingRequest {
let request = Request::new(LoadPolicyMappingRequest {
user_or_group: user_or_group.to_string(),
user_type,
is_group,
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.load_policy_mapping(request).await?.into_inner();
if !response.success {
@@ -1306,10 +1273,9 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(DeleteUserRequest {
let request = Request::new(DeleteUserRequest {
access_key: access_key.to_string(),
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.delete_user(request).await?.into_inner();
if !response.success {
@@ -1329,10 +1295,9 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(DeleteServiceAccountRequest {
let request = Request::new(DeleteServiceAccountRequest {
access_key: access_key.to_string(),
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.delete_service_account(request).await?.into_inner();
if !response.success {
@@ -1352,11 +1317,10 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadUserRequest {
let request = Request::new(LoadUserRequest {
access_key: access_key.to_string(),
temp,
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.load_user(request).await?.into_inner();
if !response.success {
@@ -1376,10 +1340,9 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadServiceAccountRequest {
let request = Request::new(LoadServiceAccountRequest {
access_key: access_key.to_string(),
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.load_service_account(request).await?.into_inner();
if !response.success {
@@ -1399,10 +1362,9 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadGroupRequest {
let request = Request::new(LoadGroupRequest {
group: group.to_string(),
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.load_group(request).await?.into_inner();
if !response.success {
@@ -1422,8 +1384,7 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(ReloadSiteReplicationConfigRequest {});
set_tonic_mutation_body_digest(&mut request)?;
let request = Request::new(ReloadSiteReplicationConfigRequest {});
let response = client.reload_site_replication_config(request).await?.into_inner();
if !response.success {
@@ -1447,10 +1408,9 @@ impl PeerRestClient {
vars.insert(PEER_RESTSIGNAL.to_string(), sig.to_string());
vars.insert(PEER_RESTSUB_SYS.to_string(), sub_sys.to_string());
vars.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string());
let mut request = Request::new(SignalServiceRequest {
let request = Request::new(SignalServiceRequest {
vars: Some(Mss { value: vars }),
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.signal_service(request).await?.into_inner();
if !response.success {
@@ -1519,8 +1479,7 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(ReloadPoolMetaRequest {});
set_tonic_mutation_body_digest(&mut request)?;
let request = Request::new(ReloadPoolMetaRequest {});
let response = client.reload_pool_meta(request).await?.into_inner();
if !response.success {
@@ -1541,10 +1500,9 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(StopRebalanceRequest {
let request = Request::new(StopRebalanceRequest {
expected_rebalance_id: expected_rebalance_id.unwrap_or_default().to_string(),
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.stop_rebalance(request).await?.into_inner();
if !response.success {
@@ -1565,8 +1523,7 @@ impl PeerRestClient {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
set_tonic_mutation_body_digest(&mut request)?;
let request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
let response = client.load_rebalance_meta(request).await?.into_inner();
@@ -1605,8 +1562,7 @@ impl PeerRestClient {
})
.collect::<Result<Vec<_>>>()?;
let mut client = self.get_client().await?;
let mut request = Request::new(StartDecommissionRequest { pool_indices });
set_tonic_mutation_body_digest(&mut request)?;
let request = Request::new(StartDecommissionRequest { pool_indices });
let response = client.start_decommission(request).await?.into_inner();
if !response.success {
@@ -1629,8 +1585,7 @@ impl PeerRestClient {
let pool_index = u32::try_from(pool_index)
.map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
let mut client = self.get_client().await?;
let mut request = Request::new(CancelDecommissionRequest { pool_index });
set_tonic_mutation_body_digest(&mut request)?;
let request = Request::new(CancelDecommissionRequest { pool_index });
let response = client.cancel_decommission(request).await?.into_inner();
if !response.success {
@@ -1653,8 +1608,7 @@ impl PeerRestClient {
let pool_index = u32::try_from(pool_index)
.map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
let mut client = self.get_client().await?;
let mut request = Request::new(ClearDecommissionRequest { pool_index });
set_tonic_mutation_body_digest(&mut request)?;
let request = Request::new(ClearDecommissionRequest { pool_index });
let response = client.clear_decommission(request).await?.into_inner();
if !response.success {
@@ -1674,14 +1628,10 @@ impl PeerRestClient {
pub async fn load_transition_tier_config(&self) -> Result<()> {
match self.load_transition_tier_config_outcome().await {
TierConfigReloadOutcome::Success => Ok(()),
// Only a reconnect-class failure says anything about the channel.
// `finalize_result` marks the peer offline and evicts its connection
// whenever the message looks network-like, and a peer that answered
// and rejected the apply can easily report one ("release RPC failed:
// transport error"). Routing those through here would gate a healthy,
// responding peer out of every unrelated RPC.
TierConfigReloadOutcome::TransientReconnect(err) => self.finalize_result(Err(err)).await,
TierConfigReloadOutcome::TransientRetrySameChannel(err) | TierConfigReloadOutcome::Terminal(err) => Err(err),
TierConfigReloadOutcome::TransientReconnect(err) | TierConfigReloadOutcome::TransientRetrySameChannel(err) => {
self.finalize_result(Err(err)).await
}
TierConfigReloadOutcome::Terminal(err) => Err(err),
}
}
@@ -1707,9 +1657,6 @@ impl PeerRestClient {
Err(err) => return tier_config_reload_connection_outcome(err),
};
let mut request = Request::new(LoadTransitionTierConfigRequest {});
if let Err(err) = set_tonic_mutation_body_digest(&mut request) {
return TierConfigReloadOutcome::Terminal(Error::other(err));
}
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
let response = match client.load_transition_tier_config(request).await {
@@ -1763,24 +1710,13 @@ fn is_tier_config_reload_connection_failure(err: &Error) -> bool {
message_has_network_needle(&message)
}
/// Classifies a reload the peer answered but refused to apply.
///
/// The peer replied, so the channel is healthy and only the remote apply
/// failed. Those failures are transient by nature: the reload reads the tier
/// mutation intents and takes the distributed tier-config lock, both of which
/// fail while any other node is restarting or while the lock quorum is briefly
/// disturbed. Retiring the worker on the first such rejection leaves that peer
/// pinned to the old configuration with nothing left to heal it, so it answers
/// `TierNotFound` for a tier the rest of the cluster already committed until a
/// second admin mutation happens to spawn a fresh worker.
///
/// Convergence is the whole point of this path, so a rejection is retried on
/// the same channel. The worker's exponential backoff caps the cost at one
/// reload every `TIER_CONFIG_RELOAD_RETRY_CAP`, and `Terminal` stays reachable
/// for transport and gRPC status failures, which is where a genuinely
/// unrecoverable peer surfaces.
fn tier_config_reload_remote_failure(error_info: Option<String>) -> TierConfigReloadOutcome {
TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info.unwrap_or_default()))
let error_info = error_info.unwrap_or_default();
if matches!(error_info.as_str(), "errServerNotInitialized" | "ServerNotInitialized") {
TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info))
} else {
TierConfigReloadOutcome::Terminal(Error::other(error_info))
}
}
fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadOutcome {
@@ -1790,14 +1726,6 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
TierConfigReloadOutcome::TransientReconnect(status.into())
} else if status.code() == Code::Unknown && status.message().starts_with("Service was not ready:") {
TierConfigReloadOutcome::TransientRetrySameChannel(status.into())
} else if status.code() == Code::Unknown
&& is_tier_config_reload_connection_failure(&Error::other(status.message().to_string()))
{
// tonic reports a connection dropped mid-call as `Unknown` carrying the
// transport error text rather than as `Unavailable`, which is what a peer
// restarting under an active mutation produces. Reconnect and retry, so
// the restart does not permanently retire this peer's reload worker.
TierConfigReloadOutcome::TransientReconnect(status.into())
} else {
TierConfigReloadOutcome::Terminal(status.into())
}
@@ -2351,12 +2279,9 @@ mod tests {
tier_config_reload_status_outcome(tonic::Status::cancelled("request cancelled")),
TierConfigReloadOutcome::Terminal(_)
));
// A peer that answered and then refused the apply is retried rather than
// retired: the channel is healthy, so the rejection reflects remote state
// that the next attempt can find healed.
assert!(matches!(
tier_config_reload_remote_failure(Some("backend unavailable".to_string())),
TierConfigReloadOutcome::TransientRetrySameChannel(_)
TierConfigReloadOutcome::Terminal(_)
));
assert!(matches!(
tier_config_reload_remote_failure(Some("errServerNotInitialized".to_string())),
@@ -2380,50 +2305,6 @@ mod tests {
));
}
/// A tier mutation issued while another node restarts must still converge on
/// the nodes that stayed up. Those peers answer the reload RPC and reject the
/// apply, because reloading reads the tier mutation intents and takes the
/// distributed tier-config lock while the lock quorum is still disturbed.
/// Classifying those rejections as terminal retired the reload worker on its
/// first attempt and pinned the peer to the previous configuration, so it
/// served `TierNotFound` for an already-committed tier until an unrelated
/// second admin mutation spawned a new worker.
#[test]
fn tier_config_reload_retries_peers_that_reject_the_apply_mid_restart() {
for error_info in [
"Lock acquisition timeout for resource '.rustfs.sys/config/tier-config.bin.lock' after 5s",
"Resource '.rustfs.sys/config/tier-config.bin.lock' is already locked by node-3",
"Internal error: release RPC failed: transport error",
"save_config_with_opts: err: PreconditionFailed",
"erasure read quorum",
] {
assert!(
matches!(
tier_config_reload_remote_failure(Some(error_info.to_string())),
TierConfigReloadOutcome::TransientRetrySameChannel(_)
),
"a peer that rejected the apply must stay retryable so it converges: {error_info}"
);
}
// An absent error message is still a rejection, not a reason to stop.
assert!(matches!(
tier_config_reload_remote_failure(None),
TierConfigReloadOutcome::TransientRetrySameChannel(_)
));
// tonic surfaces a connection dropped mid-call as `Unknown`, not `Unavailable`.
assert!(matches!(
tier_config_reload_status_outcome(tonic::Status::unknown("transport error")),
TierConfigReloadOutcome::TransientReconnect(_)
));
// An `Unknown` that is not transport-shaped stays terminal.
assert!(matches!(
tier_config_reload_status_outcome(tonic::Status::unknown("peer response unknown")),
TierConfigReloadOutcome::Terminal(_)
));
}
#[tokio::test]
async fn tier_config_reload_single_attempt_clears_offline_gate_without_redial() {
let client = test_peer_client();
@@ -2500,22 +2381,6 @@ mod tests {
}
}
#[test]
fn remote_version_state_capability_decoder_fails_closed() {
let epoch = Uuid::new_v4();
let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
.expect("small capability response should encode");
assert_eq!(
decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
epoch
);
assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
.expect("small capability response should encode");
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
}
struct TierMutationResponseFixture<'a> {
version: u32,
phase: TierMutationRpcPhase,
@@ -2787,11 +2652,6 @@ mod tests {
.with_span_list(true),
);
let _guard = tracing::subscriber::set_default(subscriber);
// The `recovery-monitor` callsite is shared with the production
// `mark_offline_and_spawn_recovery` path that sibling tests exercise from
// subscriber-less threads; without this the span can be cached as
// `Interest::never()` and silently degrade to `Span::none()`.
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
let client = test_peer_client();
let span = tracing::info_span!("request-span", request_id = "req-peer-rest");
+20 -228
View File
@@ -14,10 +14,8 @@
use crate::bucket::metadata_sys;
use crate::cluster::rpc::client::{
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
node_service_time_out_client,
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
};
use crate::cluster::rpc::set_tonic_mutation_body_digest;
use crate::disk::error::DiskError;
use crate::disk::error::{Error, Result};
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
@@ -41,84 +39,16 @@ use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClie
use rustfs_protos::proto_gen::node_service::{
DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
};
#[cfg(test)]
use std::sync::{
Mutex as StdMutex,
atomic::{AtomicBool, Ordering},
};
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
#[cfg(test)]
use tokio::sync::Notify;
use tokio::{net::TcpStream, sync::RwLock, time};
use tokio_util::sync::CancellationToken;
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::{debug, info, warn};
type Client = Arc<Box<dyn PeerS3Client>>;
#[cfg(test)]
#[derive(Default)]
pub(crate) struct DeleteBucketEmptyScanBarrier {
arrived: AtomicBool,
arrived_notify: Notify,
released: AtomicBool,
release_notify: Notify,
}
#[cfg(test)]
impl DeleteBucketEmptyScanBarrier {
pub(crate) async fn wait_until_paused(&self) {
loop {
let notified = self.arrived_notify.notified();
if self.arrived.load(Ordering::Acquire) {
return;
}
notified.await;
}
}
pub(crate) fn release(&self) {
self.released.store(true, Ordering::Release);
self.release_notify.notify_waiters();
}
async fn pause(&self) {
self.arrived.store(true, Ordering::Release);
self.arrived_notify.notify_waiters();
loop {
let notified = self.release_notify.notified();
if self.released.load(Ordering::Acquire) {
return;
}
notified.await;
}
}
}
#[cfg(test)]
static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
#[cfg(test)]
pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
*DELETE_BUCKET_EMPTY_SCAN_BARRIER
.lock()
.expect("empty scan barrier lock should not be poisoned") = Some(barrier.clone());
barrier
}
#[cfg(test)]
async fn pause_after_delete_bucket_empty_scan() {
let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
.lock()
.expect("empty scan barrier lock should not be poisoned")
.take();
if let Some(barrier) = barrier {
barrier.pause().await;
}
}
#[derive(Clone, Debug)]
pub struct ScannerBucketListing {
pub buckets: Vec<BucketInfo>,
@@ -159,22 +89,6 @@ fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Erro
reduce_write_quorum_errs(per_pool_errs, BUCKET_OP_IGNORED_ERRS, pool_write_quorum(per_pool_errs.len()))
}
fn resolve_heal_bucket_mode(opts: &mut HealOpts, pool_errs: &[Option<Error>]) -> Result<()> {
if opts.recreate {
return Ok(());
}
if let Some(err) = pool_errs
.iter()
.flatten()
.find(|err| **err != Error::DiskNotFound && **err != Error::VolumeNotFound)
{
return Err(err.clone());
}
opts.remove = is_all_buckets_not_found(pool_errs);
opts.recreate = !opts.remove;
Ok(())
}
#[async_trait]
pub trait PeerS3Client: Debug + Sync + Send + 'static {
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
@@ -245,7 +159,10 @@ impl S3PeerSys {
pool_errs.push(reduce_pool_write_quorum_errs(&per_pool_errs));
}
resolve_heal_bucket_mode(&mut opts, &pool_errs)?;
if !opts.recreate {
opts.remove = is_all_buckets_not_found(&pool_errs);
opts.recreate = !opts.remove;
}
let mut futures = Vec::new();
let heal_bucket_results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.clients.len()]));
@@ -719,22 +636,24 @@ impl PeerS3Client for LocalPeerS3Client {
return Err(Error::ErasureWriteQuorum);
}
if opts.force_if_empty && !opts.force {
let force = if opts.force_if_empty && !opts.force {
for disk in local_disks.iter() {
if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? {
return Err(Error::VolumeNotEmpty);
}
}
#[cfg(test)]
pause_after_delete_bucket_empty_scan().await;
}
true
} else {
opts.force
};
let mut futures = Vec::with_capacity(local_disks.len());
for disk in local_disks.iter() {
// `force_if_empty` is validation-only. Passing it as force would let
// a PutObject committed after the scan be removed recursively.
futures.push(disk.delete_volume(bucket, opts.force));
// Non-force delete refuses a non-empty bucket (VolumeNotEmpty), which
// the recreate loop below turns into BucketNotEmpty; only an explicit
// force delete removes recursively (backlog#799 B1).
futures.push(disk.delete_volume(bucket, force));
}
let results = join_all(futures).await;
@@ -797,15 +716,6 @@ pub struct RemotePeerS3Client {
}
impl RemotePeerS3Client {
fn encode_delete_bucket_options(opts: &DeleteBucketOptions) -> Result<String> {
let mut remote_opts = opts.clone();
// Older peers promote `force_if_empty` to recursive force after their
// metadata scan. Keep this coordinator-only hint off the wire so a
// mixed-version delete fails closed on non-empty directory remnants.
remote_opts.force_if_empty = false;
serde_json::to_string(&remote_opts).map_err(Into::into)
}
fn recovery_monitor_span(addr: &str) -> tracing::Span {
tracing::info_span!(
"recovery-monitor",
@@ -832,7 +742,7 @@ impl RemotePeerS3Client {
client
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
@@ -1007,11 +917,10 @@ impl PeerS3Client for RemotePeerS3Client {
|| async {
let options: String = serde_json::to_string(opts)?;
let mut client = self.get_client().await?;
let mut request = Request::new(HealBucketRequest {
let request = Request::new(HealBucketRequest {
bucket: bucket.to_string(),
options,
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.heal_bucket(request).await?.into_inner();
if !response.success {
return if let Some(err) = response.error {
@@ -1064,11 +973,10 @@ impl PeerS3Client for RemotePeerS3Client {
|| async {
let options = serde_json::to_string(opts)?;
let mut client = self.get_client().await?;
let mut request = Request::new(MakeBucketRequest {
let request = Request::new(MakeBucketRequest {
name: bucket.to_string(),
options,
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.make_bucket(request).await?.into_inner();
if !response.success {
@@ -1116,14 +1024,13 @@ impl PeerS3Client for RemotePeerS3Client {
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
self.execute_with_timeout(
|| async {
let options = Self::encode_delete_bucket_options(opts)?;
let options = serde_json::to_string(opts)?;
let mut client = self.get_client().await?;
let mut request = Request::new(DeleteBucketRequest {
let request = Request::new(DeleteBucketRequest {
bucket: bucket.to_string(),
options,
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.delete_bucket(request).await?.into_inner();
if !response.success {
return if let Some(err) = response.error {
@@ -1490,49 +1397,6 @@ mod tests {
}
}
#[test]
fn remote_delete_bucket_options_fail_closed_for_legacy_peers() {
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
no_lock: true,
no_recreate: true,
force_if_empty: true,
..Default::default()
})
.expect("remote delete options should serialize");
let legacy_opts: DeleteBucketOptions =
serde_json::from_str(&encoded).expect("legacy peer should decode remote delete options");
assert!(legacy_opts.no_lock);
assert!(legacy_opts.no_recreate);
assert!(!legacy_opts.force);
assert!(!legacy_opts.force_if_empty);
let legacy_recursive_force = if legacy_opts.force_if_empty && !legacy_opts.force {
true
} else {
legacy_opts.force
};
assert!(
!legacy_recursive_force,
"legacy peer must not upgrade empty-only delete to recursive force"
);
}
#[test]
fn remote_delete_bucket_options_preserve_explicit_force() {
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
force: true,
force_if_empty: true,
..Default::default()
})
.expect("remote force-delete options should serialize");
let remote_opts: DeleteBucketOptions =
serde_json::from_str(&encoded).expect("remote peer should decode force-delete options");
assert!(remote_opts.force);
assert!(!remote_opts.force_if_empty);
}
#[tokio::test]
async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
let client = test_remote_peer("http://peer-network-error:9000");
@@ -1690,54 +1554,6 @@ mod tests {
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn local_peer_force_if_empty_preserves_unclassified_file_in_selected_pool() {
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for empty-only delete regression");
let disks = init_test_local_disks_for_pools(
&temp_dir,
&[(0, 1), (1, 1)],
"local-peer-force-if-empty-preserves-unclassified-file",
)
.await;
let bucket = "empty-only-delete-bucket";
let marker = "object/commit-marker";
let data = bytes::Bytes::from_static(b"committed object data");
disks[1]
.make_volume(bucket)
.await
.expect("bucket should be created in the selected pool");
disks[1]
.write_all(bucket, marker, data.clone())
.await
.expect("unclassified committed file should be written");
let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
.delete_bucket(
bucket,
&DeleteBucketOptions {
force_if_empty: true,
..Default::default()
},
)
.await
.expect_err("empty-only delete must not recursively remove an unclassified file");
assert_eq!(err, Error::VolumeNotEmpty);
assert_eq!(
disks[1]
.read_all(bucket, marker)
.await
.expect("unclassified committed file should be preserved"),
data
);
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn heal_bucket_local_recreates_missing_bucket_volumes() {
@@ -1802,30 +1618,6 @@ mod tests {
assert_eq!(err, Error::VolumeExists);
}
#[test]
fn heal_bucket_mode_fails_closed_on_incomplete_topology() {
let mut opts = HealOpts::default();
assert_eq!(
resolve_heal_bucket_mode(&mut opts, &[Some(Error::ErasureWriteQuorum)]),
Err(Error::ErasureWriteQuorum)
);
assert!(!opts.recreate);
assert!(!opts.remove);
}
#[test]
fn heal_bucket_mode_distinguishes_deleted_and_partial_buckets() {
let mut deleted = HealOpts::default();
resolve_heal_bucket_mode(&mut deleted, &[Some(Error::VolumeNotFound)]).unwrap();
assert!(deleted.remove);
assert!(!deleted.recreate);
let mut partial = HealOpts::default();
resolve_heal_bucket_mode(&mut partial, &[None, Some(Error::VolumeNotFound)]).unwrap();
assert!(!partial.remove);
assert!(partial.recreate);
}
#[tokio::test]
async fn test_make_bucket_reduces_quorum_by_pool_participants() {
let peer_sys = S3PeerSys {
+8 -160
View File
@@ -13,8 +13,8 @@
// limitations under the License.
use crate::cluster::rpc::client::{
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
};
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
use crate::cluster::rpc::internode_data_transport::{
@@ -25,7 +25,7 @@ use crate::disk::error::{Error, Result};
use crate::disk::{
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
disk_store::{
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -50,9 +50,8 @@ use rustfs_protos::proto_gen::node_service::{
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, node_service_client::NodeServiceClient,
RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest,
WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
};
use serde::{Serialize, de::DeserializeOwned};
use std::{
@@ -71,7 +70,7 @@ use tokio::{
time::timeout,
};
use tokio_util::sync::CancellationToken;
use tonic::{Code, Request, service::interceptor::InterceptedService};
use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
use tracing::{debug, trace, warn};
use uuid::Uuid;
@@ -101,18 +100,6 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
if response.protocol_version != SNAPSHOT_LEASE_PROTOCOL_VERSION {
return Err(Error::other("remote snapshot lease protocol is incompatible"));
}
SnapshotLeaseToken::from_slice(&response.token)
}
/// Bind a mutating disk RPC to its canonical body: the digest lands in the request metadata, and
/// the signing interceptor folds it (plus a replay-protected nonce) into the v2 signature scope
@@ -1083,7 +1070,7 @@ impl RemoteDisk {
internode_offline_bypass_reason(&self.addr).map(Error::other)
}
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
if let Some(err) = self.offline_bypass_error() {
return Err(err);
}
@@ -1096,7 +1083,7 @@ impl RemoteDisk {
/// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
/// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
/// is disabled.
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
if let Some(err) = self.offline_bypass_error() {
return Err(err);
}
@@ -1797,81 +1784,6 @@ impl DiskAPI for RemoteDisk {
.await
}
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
self.execute_with_timeout(
|| async {
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(SnapshotLeaseRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
.map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
});
let canonical_body = rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "acquire_snapshot_lease")?;
let response = client.acquire_snapshot_lease(request).await?.into_inner();
snapshot_lease_token_from_response(response)
},
get_max_timeout_duration(),
)
.await
}
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
self.execute_with_timeout(
|| async {
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(SnapshotLeaseRenewRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
token: token.as_bytes().to_vec().into(),
ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
.map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
});
let canonical_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "renew_snapshot_lease")?;
let response = client.renew_snapshot_lease(request).await?.into_inner();
snapshot_lease_token_from_response(response)
},
get_max_timeout_duration(),
)
.await
}
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
self.execute_with_timeout(
|| async {
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(SnapshotLeaseReleaseRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
token: token.as_bytes().to_vec().into(),
});
let canonical_body = rustfs_protos::canonical_snapshot_lease_release_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "release_snapshot_lease")?;
let response = client.release_snapshot_lease(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
trace!(
@@ -3005,7 +2917,6 @@ mod tests {
use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport};
use crate::runtime::sources as runtime_sources;
use serde_json::Value;
use serial_test::serial;
use std::io::{self as std_io, Write};
use std::pin::Pin;
use std::sync::{Arc, Mutex, Mutex as StdMutex, Once};
@@ -3019,48 +2930,6 @@ mod tests {
static INIT: Once = Once::new();
// `#[serial(internode_metrics)]` marks every test that observes
// `global_internode_metrics()`. Those counters are a process-wide singleton:
// some of these tests snapshot a counter, run one decode, and assert on the
// delta, while others deliberately record decode errors or call
// `reset_internode_metrics_for_test()`. Run concurrently in one process they
// corrupt each other's deltas — a sibling's error bumps the "no decode error"
// assertion off zero, and a sibling's reset can drive an `after > before`
// assertion backwards.
//
// The marker only takes effect under the `cargo test` fallback; nextest
// already isolates each test in its own process, so every test there gets its
// own copy of the counters (see `docs/testing/README.md`). Any new test that
// reads or mutates the global internode metrics belongs in this group.
#[test]
fn snapshot_lease_response_requires_current_protocol_and_valid_token() {
let token = SnapshotLeaseToken::new();
let response = SnapshotLeaseResponse {
success: true,
token: token.as_bytes().to_vec().into(),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: None,
};
assert_eq!(snapshot_lease_token_from_response(response).unwrap(), token);
let incompatible = SnapshotLeaseResponse {
success: true,
token: token.as_bytes().to_vec().into(),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION + 1,
error: None,
};
assert!(snapshot_lease_token_from_response(incompatible).is_err());
let malformed = SnapshotLeaseResponse {
success: true,
token: Bytes::from_static(b"not-a-uuid"),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: None,
};
assert!(snapshot_lease_token_from_response(malformed).is_err());
}
#[test]
fn list_volumes_decode_rejects_a_malformed_entry() {
let valid = serde_json::to_string(&VolumeInfo {
@@ -3321,7 +3190,6 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn read_multiple_response_decode_prefers_msgpack_payloads() {
let endpoint = sample_remote_endpoint();
let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary");
@@ -3344,7 +3212,6 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn read_multiple_response_decode_falls_back_to_json_payloads() {
let endpoint = sample_remote_endpoint();
let json_resp = sample_read_multiple_resp("json", b"fallback");
@@ -3366,7 +3233,6 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn rename_data_response_accepts_legacy_json_without_decode_error() {
crate::cluster::rpc::runtime_sources::reset_internode_metrics_for_test();
let response = RenameDataResp {
@@ -3518,7 +3384,6 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
let endpoint = sample_remote_endpoint();
let response = ReadMultipleResponse {
@@ -3544,7 +3409,6 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn read_multiple_response_decode_reports_corrupt_json_item() {
let endpoint = sample_remote_endpoint();
let response = ReadMultipleResponse {
@@ -3585,7 +3449,6 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn batch_read_version_response_decode_prefers_msgpack_payloads() {
let endpoint = sample_remote_endpoint();
let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true);
@@ -3606,7 +3469,6 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn batch_read_version_response_rejects_invalid_success_metadata() {
let endpoint = sample_remote_endpoint();
let mut response_item = sample_batch_read_version_resp(0, "invalid-object", true);
@@ -3626,7 +3488,6 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn batch_read_version_response_decode_reports_corrupt_msgpack_item() {
let endpoint = sample_remote_endpoint();
let response = BatchReadVersionResponse {
@@ -3653,7 +3514,6 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn batch_read_version_response_decode_reports_corrupt_json_item() {
let endpoint = sample_remote_endpoint();
let response = BatchReadVersionResponse {
@@ -4443,7 +4303,6 @@ mod tests {
}
#[tokio::test]
#[serial(internode_metrics)]
async fn test_remote_disk_create_file_retries_once_on_retryable_open_write_error() {
let transport = RetryingOpenWriteInternodeDataTransport::with_steps(vec![
OpenWriteTestStep::Error(DiskError::from(rustfs_rio::new_test_internode_http_io_error(
@@ -4482,7 +4341,6 @@ mod tests {
}
#[tokio::test]
#[serial(internode_metrics)]
async fn test_remote_disk_read_file_stream_retries_once_on_retryable_open_read_error() {
// A transient reset-by-peer on a shard read during the read-after-write window must be
// absorbed by one re-dial rather than eroding read quorum (issue #2761).
@@ -5332,11 +5190,6 @@ mod tests {
.with_span_list(true),
);
let _guard = tracing::subscriber::set_default(subscriber);
// The `recovery-monitor` span and the monitor's own log events are
// production callsites that sibling tests exercise from subscriber-less
// threads; without this they can be cached as `Interest::never()` and go
// silently missing here.
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
let endpoint = Endpoint {
url: url::Url::parse("http://127.0.0.1:59996/data").expect("endpoint URL should parse"),
@@ -5391,11 +5244,6 @@ mod tests {
.with_span_list(true),
);
let _guard = tracing::subscriber::set_default(subscriber);
// The `recovery-monitor` span and the monitor's own log events are
// production callsites that sibling tests exercise from subscriber-less
// threads; without this they can be cached as `Interest::never()` and go
// silently missing here.
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
let addr = "http://127.0.0.1:59997".to_string();
let endpoint = Endpoint {
+11 -21
View File
@@ -12,10 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::client::{
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
};
use crate::cluster::rpc::set_tonic_mutation_body_digest;
use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use async_trait::async_trait;
use bytes::Bytes;
use rustfs_lock::{
@@ -31,6 +28,7 @@ use std::time::Duration;
use tokio::time::timeout;
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::{debug, info, warn};
/// Remote lock client implementation
@@ -79,7 +77,7 @@ impl RemoteClient {
}
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
// P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
// offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
// change quorum; the self-healing re-probe keeps the peer recoverable.
@@ -315,11 +313,10 @@ impl LockClient for RemoteClient {
info!("remote acquire_exclusive for {}", request.resource);
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
set_tonic_mutation_body_digest(&mut req)?;
let resp = match self.execute_rpc("lock", &resource_summary, client.lock(req)).await {
Ok(resp) => resp.into_inner(),
@@ -350,7 +347,7 @@ impl LockClient for RemoteClient {
let mut client = self.get_client().await?;
let resource_summary = Self::summarize_resources(requests);
let mut req = Request::new(BatchGenerallyLockRequest {
let req = Request::new(BatchGenerallyLockRequest {
args: requests
.iter()
.map(|request| {
@@ -358,7 +355,6 @@ impl LockClient for RemoteClient {
})
.collect::<Result<Vec<_>>>()?,
});
set_tonic_mutation_body_digest(&mut req)?;
let resp = match self
.execute_rpc("lock_batch", &resource_summary, client.lock_batch(req))
@@ -399,8 +395,7 @@ impl LockClient for RemoteClient {
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
let mut client = self.get_client().await?;
let resource_summary = unlock_request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest { args: request_string });
set_tonic_mutation_body_digest(&mut req)?;
let req = Request::new(GenerallyLockRequest { args: request_string });
let resp = self
.execute_rpc("release", &resource_summary, client.un_lock(req))
.await?
@@ -419,7 +414,7 @@ impl LockClient for RemoteClient {
let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
let mut client = self.get_client().await?;
let resource_summary = Self::summarize_resources(&unlock_requests);
let mut req = Request::new(BatchGenerallyLockRequest {
let req = Request::new(BatchGenerallyLockRequest {
args: unlock_requests
.iter()
.map(|request| {
@@ -427,7 +422,6 @@ impl LockClient for RemoteClient {
})
.collect::<Result<Vec<_>>>()?,
});
set_tonic_mutation_body_digest(&mut req)?;
let resp = self
.execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
@@ -446,11 +440,10 @@ impl LockClient for RemoteClient {
let refresh_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let resource_summary = refresh_request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&refresh_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
set_tonic_mutation_body_digest(&mut req)?;
let resp = self
.execute_rpc("refresh", &resource_summary, client.refresh(req))
.await?
@@ -466,11 +459,10 @@ impl LockClient for RemoteClient {
let force_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let resource_summary = force_request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&force_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
set_tonic_mutation_body_digest(&mut req)?;
let resp = self
.execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
.await?
@@ -491,11 +483,10 @@ impl LockClient for RemoteClient {
let mut client = self.get_client().await?;
// Try to acquire a very short-lived lock to test availability
let mut req = Request::new(GenerallyLockRequest {
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
set_tonic_mutation_body_digest(&mut req)?;
// Try exclusive lock first with very short timeout
let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
@@ -506,11 +497,10 @@ impl LockClient for RemoteClient {
if resp.success {
// If we successfully acquired the lock, the resource was free.
// Immediately release it on a best-effort basis.
let mut release_req = Request::new(GenerallyLockRequest {
let release_req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
set_tonic_mutation_body_digest(&mut release_req)?;
let _ = self
.execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
.await;
-8
View File
@@ -1365,14 +1365,6 @@ impl DiskAPI for LocalDiskWrapper {
.await
}
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
self.track_disk_health(
|| async { self.disk.renew_snapshot_lease(volume, path, token).await },
get_max_timeout_duration(),
)
.await
}
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
self.track_disk_health(
|| async { self.disk.delete_data_dir(volume, path, opts).await },
File diff suppressed because it is too large Load Diff
-22
View File
@@ -79,18 +79,6 @@ impl SnapshotLeaseToken {
pub fn new() -> Self {
Self(Uuid::new_v4())
}
pub fn from_slice(bytes: &[u8]) -> Result<Self> {
let uuid = Uuid::from_slice(bytes).map_err(|_| Error::other("invalid snapshot lease token"))?;
if uuid.is_nil() {
return Err(Error::other("invalid snapshot lease token"));
}
Ok(Self(uuid))
}
pub fn as_bytes(&self) -> &[u8; 16] {
self.0.as_bytes()
}
}
impl Default for SnapshotLeaseToken {
@@ -296,13 +284,6 @@ impl DiskAPI for Disk {
}
}
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
match self {
Disk::Local(local_disk) => local_disk.renew_snapshot_lease(volume, path, token).await,
Disk::Remote(remote_disk) => remote_disk.renew_snapshot_lease(volume, path, token).await,
}
}
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
match self {
Disk::Local(local_disk) => local_disk.delete_data_dir(volume, path, opts).await,
@@ -713,9 +694,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
async fn release_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<()> {
Err(Error::other("snapshot leases are not supported by this disk"))
}
async fn renew_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
Err(Error::other("snapshot leases are not supported by this disk"))
}
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
self.delete(volume, path, opts).await?;
Ok(DataDirDeleteStatus::Deleted)
+9 -321
View File
@@ -25,7 +25,7 @@ use std::{
sync::{Arc, LazyLock, Weak},
};
use tokio::fs;
use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit};
use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
use tracing::warn;
/// Check path length according to OS limits.
@@ -123,8 +123,6 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
let cpu_scaled = cpu_count
@@ -159,24 +157,6 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
limiter
}
/// Serialize a bucket's local metadata commits with physical bucket removal.
///
/// The key includes the canonical disk root, so independently reconnected
/// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
/// disconnected disks do not keep the registry alive.
pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
let key = root.join(volume);
let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
return lock;
}
let lock = Arc::new(RwLock::new(()));
locks.insert(key, Arc::downgrade(&lock));
lock
}
/// Always acquire the per-disk permit before the process-wide permit. Keeping
/// this order uniform prevents one slow disk from reserving global capacity
/// while it waits for its own concurrency slot.
@@ -592,14 +572,15 @@ async fn reliable_rename_inner(
base_dir: impl AsRef<Path>,
warn_on_missing_source: bool,
) -> io::Result<()> {
let parent_guard = match dst_file_path.as_ref().parent() {
Some(parent) => Some(mkdir_all_below_existing_base(parent, base_dir.as_ref()).await?),
None => None,
};
if let Some(parent) = dst_file_path.as_ref().parent()
&& !file_exists(parent)
{
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
}
let mut i = 0;
loop {
if let Err(e) = rename_into_existing_parent(src_file_path.as_ref(), dst_file_path.as_ref(), parent_guard.as_ref()) {
if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
if should_retry_rename(&e, i) {
i += 1;
continue;
@@ -616,159 +597,6 @@ async fn reliable_rename_inner(
Ok(())
}
#[cfg(unix)]
fn rename_into_existing_parent(
src_file_path: &Path,
dst_file_path: &Path,
parent_guard: Option<&ExistingBaseDirectoryGuard>,
) -> io::Result<()> {
use rustix::fs::{Mode, OFlags, open, renameat};
let Some(parent_guard) = parent_guard else {
return super::fs::rename_std(src_file_path, dst_file_path);
};
let src_parent = src_file_path
.parent()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
let src_name = src_file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a file name"))?;
let dst_name = dst_file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?;
let src_parent = open(
src_parent,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)
.map_err(io::Error::from)?;
let dst_parent = parent_guard
.last()
.ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?;
renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from)
}
#[cfg(not(unix))]
fn rename_into_existing_parent(
src_file_path: &Path,
dst_file_path: &Path,
_parent_guard: Option<&ExistingBaseDirectoryGuard>,
) -> io::Result<()> {
super::fs::rename_std(src_file_path, dst_file_path)
}
async fn mkdir_all_below_existing_base(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
let dir_path = dir_path.to_path_buf();
let base_dir = base_dir.to_path_buf();
tokio::task::spawn_blocking(move || mkdir_all_below_existing_base_std(&dir_path, &base_dir)).await?
}
#[cfg(windows)]
pub(crate) type ExistingBaseDirectoryGuard = Vec<winapi_util::Handle>;
#[cfg(unix)]
pub(crate) type ExistingBaseDirectoryGuard = Vec<std::os::fd::OwnedFd>;
#[cfg(all(not(unix), not(windows)))]
pub(crate) type ExistingBaseDirectoryGuard = ();
#[cfg(windows)]
fn lock_windows_directory(path: &Path) -> io::Result<winapi_util::Handle> {
use std::os::windows::fs::OpenOptionsExt;
const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
const FILE_SHARE_READ: u32 = 0x1;
let file = std::fs::OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ)
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
.open(path)?;
let handle = winapi_util::Handle::from_file(file);
let info = winapi_util::file::information(&handle)?;
if info.file_attributes() & FILE_ATTRIBUTE_DIRECTORY == 0
|| info.file_attributes() & u64::from(FILE_ATTRIBUTE_REPARSE_POINT) != 0
{
return Err(io::Error::from(io::ErrorKind::NotADirectory));
}
Ok(handle)
}
pub(crate) fn mkdir_all_below_existing_base_std(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
let relative = dir_path
.strip_prefix(base_dir)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?;
for component in relative.components() {
if !matches!(component, Component::Normal(_) | Component::CurDir) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"rename destination contains an invalid path component",
));
}
}
#[cfg(unix)]
{
use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
use rustix::io::Errno;
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
for component in relative.components() {
let Component::Normal(component) = component else {
continue;
};
let parent = parents
.last()
.expect("base directory guard should contain the base directory");
match mkdirat(parent, component, mode) {
Ok(()) => {}
Err(Errno::EXIST) => {}
Err(err) => return Err(err.into()),
}
parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
}
Ok(parents)
}
#[cfg(windows)]
{
let mut handles = vec![lock_windows_directory(base_dir)?];
let mut current = base_dir.to_path_buf();
for component in relative.components() {
let Component::Normal(component) = component else {
continue;
};
current.push(component);
match std::fs::create_dir(&current) {
Ok(()) => {}
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
Err(err) => return Err(err),
}
handles.push(lock_windows_directory(&current)?);
}
Ok(handles)
}
#[cfg(all(not(unix), not(windows)))]
{
let _ = relative;
Err(io::Error::new(
io::ErrorKind::Unsupported,
"safe recursive directory creation is unavailable on this platform",
))
}
}
fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
warn!(
"reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
@@ -899,20 +727,6 @@ mod tests {
Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
}
#[tokio::test]
async fn disk_volume_mutation_lock_is_shared_per_root_and_volume() {
let temp_dir = tempdir().expect("create temp dir");
let first = disk_volume_mutation_lock(temp_dir.path(), "bucket");
let second = disk_volume_mutation_lock(temp_dir.path(), "bucket");
let other = disk_volume_mutation_lock(temp_dir.path(), "other-bucket");
assert!(Arc::ptr_eq(&first, &second), "reconnected disks must share a bucket mutation lock");
assert!(!Arc::ptr_eq(&first, &other), "different buckets must not serialize each other");
let _write_guard = first.write().await;
assert!(second.try_read().is_err(), "a bucket delete lock must exclude local commits");
}
#[derive(Clone, Default)]
struct CapturedLogs {
buffer: Arc<Mutex<Vec<u8>>>,
@@ -957,26 +771,9 @@ mod tests {
}
}
/// Holds a `warn_capture()` capture alive: the thread-local subscriber, plus
/// the pin that keeps tracing's process-global callsite-interest cache from
/// being decided by some other test's thread.
struct WarnCaptureGuard {
_subscriber: tracing::subscriber::DefaultGuard,
_callsite_pin: tracing::Dispatch,
}
/// Capture WARN-level output on the current thread; tokio tests here run on
/// the current-thread runtime, so the guard covers the whole test body.
///
/// The callsite pin matters because `warn_reliable_rename_failure` is a
/// single production callsite shared with tests that call `rename_all`
/// *without* installing a subscriber — `rename_all_missing_source_returns_file_not_found`
/// is one. Whichever thread reaches it first fixes its `Interest`
/// process-wide, so without the pin that sibling can cache
/// `Interest::never()` and the WARN never fires here at all, leaving the
/// "must keep the WARN" assertions staring at empty output. See
/// [`crate::test_tracing::pin_callsite_interest_for_test`].
fn warn_capture() -> (CapturedLogs, WarnCaptureGuard) {
fn warn_capture() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
.with_max_level(tracing::Level::WARN)
@@ -984,10 +781,7 @@ mod tests {
.with_ansi(false)
.without_time()
.finish();
let guard = WarnCaptureGuard {
_subscriber: tracing::subscriber::set_default(subscriber),
_callsite_pin: crate::test_tracing::pin_callsite_interest_for_test(),
};
let guard = tracing::subscriber::set_default(subscriber);
(logs, guard)
}
@@ -1174,112 +968,6 @@ mod tests {
assert_eq!(std::fs::read(dst.join("nested").join("part.1")).expect("read moved part"), b"payload");
}
#[tokio::test]
async fn rename_all_does_not_recreate_missing_base_directory() {
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
std::fs::create_dir(&base).expect("create destination base");
let src = temp_dir.path().join("staged-object");
std::fs::write(&src, b"payload").expect("write staged object");
let dst = base.join("object").join("xl.meta");
std::fs::remove_dir(&base).expect("delete destination base before commit");
let err = rename_all(&src, &dst, &base)
.await
.expect_err("rename must not recreate a deleted destination base");
assert!(matches!(err, DiskError::FileNotFound));
assert!(src.exists(), "failed commit must preserve the staged source");
assert!(!base.exists(), "failed commit must not recreate the deleted bucket");
}
#[cfg(unix)]
#[tokio::test]
async fn rename_all_rejects_a_replaced_base_with_an_existing_parent() {
use std::os::unix::fs::symlink;
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
let outside = temp_dir.path().join("outside");
std::fs::create_dir(&base).expect("create destination base");
std::fs::create_dir_all(outside.join("object")).expect("create outside destination parent");
let src = temp_dir.path().join("staged-object");
std::fs::write(&src, b"payload").expect("write staged object");
let dst = base.join("object").join("xl.meta");
std::fs::remove_dir(&base).expect("remove destination base before replacement");
symlink(&outside, &base).expect("replace destination base with a symlink");
rename_all(&src, &dst, &base)
.await
.expect_err("rename must reject an existing destination parent below a replaced base");
assert!(src.exists(), "rejected rename must preserve the staged source");
assert!(
!outside.join("object/xl.meta").exists(),
"rename must not publish through the replacement symlink"
);
}
#[cfg(windows)]
#[test]
fn windows_parent_guard_blocks_base_and_intermediate_replacement() {
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
std::fs::create_dir(&base).expect("create destination base");
let parent = base.join("object").join("nested");
let guard = mkdir_all_below_existing_base_std(&parent, &base).expect("create and lock destination parents");
std::fs::rename(&base, temp_dir.path().join("replacement-base"))
.expect_err("the locked base must not be replaceable before commit");
std::fs::rename(base.join("object"), base.join("replacement-object"))
.expect_err("a locked intermediate directory must not be replaceable before commit");
drop(guard);
std::fs::rename(base.join("object"), base.join("replacement-object"))
.expect("replacement should succeed after the commit guard is released");
}
#[cfg(unix)]
#[tokio::test]
async fn rename_parent_creation_rejects_symlinked_base() {
use std::os::unix::fs::symlink;
let temp_dir = tempdir().expect("create temp dir");
let outside = temp_dir.path().join("outside");
std::fs::create_dir(&outside).expect("create outside directory");
let base = temp_dir.path().join("bucket");
symlink(&outside, &base).expect("create symlinked base");
mkdir_all_below_existing_base(&base.join("object"), &base)
.await
.expect_err("symlinked base must be rejected");
assert!(!outside.join("object").exists(), "parent creation must remain confined to the base");
}
#[cfg(unix)]
#[tokio::test]
async fn rename_parent_creation_rejects_symlink_below_base() {
use std::os::unix::fs::symlink;
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
let outside = temp_dir.path().join("outside");
std::fs::create_dir(&base).expect("create destination base");
std::fs::create_dir(&outside).expect("create outside directory");
symlink(&outside, base.join("linked")).expect("create symlink below base");
mkdir_all_below_existing_base(&base.join("linked/object"), &base)
.await
.expect_err("symlink below base must be rejected");
assert!(
!outside.join("object").exists(),
"parent creation must not follow a symlink outside the base"
);
}
#[tokio::test]
async fn fsync_dir_succeeds_on_directory() {
let temp_dir = tempdir().expect("create temp dir");
-3
View File
@@ -93,6 +93,3 @@ pub(crate) mod ecstore_validation_blackbox;
#[cfg(test)]
pub(crate) mod test_metrics;
#[cfg(test)]
pub(crate) mod test_tracing;
@@ -1,80 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use http::{HeaderMap, HeaderValue};
use std::collections::HashMap;
use std::error::Error;
use std::fmt::{Display, Formatter};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadEncryptionMode {
Direct { base_nonce: [u8; 12] },
Object,
}
pub struct ReadEncryptionMaterial {
pub key_bytes: [u8; 32],
pub mode: ReadEncryptionMode,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncryptionResolutionErrorKind {
InvalidRequest,
InvalidMetadata,
ServiceUnavailable,
DecryptionFailed,
}
#[derive(Debug)]
pub struct EncryptionResolutionError {
kind: EncryptionResolutionErrorKind,
message: String,
}
impl EncryptionResolutionError {
pub fn new(kind: EncryptionResolutionErrorKind, message: impl Into<String>) -> Self {
Self {
kind,
message: message.into(),
}
}
pub fn kind(&self) -> EncryptionResolutionErrorKind {
self.kind
}
}
impl Display for EncryptionResolutionError {
fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
impl Error for EncryptionResolutionError {}
pub struct ReadEncryptionRequest<'a> {
pub bucket: &'a str,
pub object: &'a str,
pub metadata: &'a HashMap<String, String>,
pub headers: &'a HeaderMap<HeaderValue>,
}
#[async_trait]
pub trait ObjectEncryptionResolver: Send + Sync {
async fn resolve_read_material(
&self,
request: ReadEncryptionRequest<'_>,
) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError>;
}
-5
View File
@@ -84,7 +84,6 @@ pub(crate) fn legacy_encrypted_range_seek_enabled() -> bool {
}
mod body_cache_hook;
mod encryption;
mod hook_slot;
mod object_mutation_hook;
mod readers;
@@ -99,10 +98,6 @@ pub use body_cache_hook::{
pub(crate) use body_cache_hook::{
get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook,
};
pub use encryption::{
EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
ReadEncryptionMode, ReadEncryptionRequest,
};
pub(crate) use object_mutation_hook::notify_object_mutation;
pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook};
pub use readers::*;
File diff suppressed because it is too large Load Diff
+46 -17
View File
@@ -273,9 +273,29 @@ impl ObjectInfo {
}
pub fn is_encrypted(&self) -> bool {
self.user_defined
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
// Corresponding to the logic in rustfs/src/sse.rs/encryption_material_to_metadata function
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
self.user_defined.keys().any(|key| {
let lower = key.to_ascii_lowercase();
lower.starts_with("x-minio-encryption-")
|| lower.starts_with("x-minio-internal-server-side-encryption-")
|| matches!(
lower.as_str(),
"x-minio-internal-encrypted-multipart"
| "x-rustfs-encryption-key"
| "x-rustfs-encryption-algorithm"
| "x-rustfs-encryption-iv"
| "x-rustfs-encryption-key-id"
| "x-rustfs-encryption-context"
| "x-rustfs-encryption-tag"
| "x-amz-server-side-encryption-aws-kms-key-id"
| SSEC_ALGORITHM_HEADER
| SSEC_KEY_HEADER
| SSEC_KEY_MD5_HEADER
| "x-amz-server-side-encryption"
)
})
}
/// Maximum inline size for non-versioned objects (128 KiB).
@@ -319,7 +339,26 @@ impl ObjectInfo {
}
pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
rustfs_utils::http::get_object_encryption_original_size(&self.user_defined)
let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
if let Some(size_str) = self
.user_defined
.get("x-rustfs-encryption-original-size")
.map(String::as_str)
.or_else(|| {
self.user_defined
.get("x-amz-server-side-encryption-customer-original-size")
.map(String::as_str)
})
.or(actual_size.as_deref())
&& !size_str.is_empty()
{
let size = size_str
.parse::<i64>()
.map_err(|e| std::io::Error::other(format!("Failed to parse encryption original size: {e}")))?;
return Ok(Some(size));
}
Ok(None)
}
pub fn decrypted_size(&self) -> std::io::Result<i64> {
@@ -349,6 +388,9 @@ impl ObjectInfo {
return Ok(actual_size);
}
// Check if object is encrypted
// Managed SSE stores original size in x-rustfs-encryption-original-size metadata
// SSE-C stores original size in x-amz-server-side-encryption-customer-original-size
if let Some(size) = self.encryption_original_size()? {
return Ok(size);
}
@@ -839,19 +881,6 @@ mod tests {
assert!(!object.is_inline_fast_path_eligible(), "transitioned objects must fall back");
}
#[test]
fn minio_internal_encryption_metadata_is_not_treated_as_plaintext() {
let object = ObjectInfo {
user_defined: Arc::new(HashMap::from([(
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key".to_string(),
"sealed".to_string(),
)])),
..Default::default()
};
assert!(object.is_encrypted());
}
#[test]
fn versions_after_marker_handles_null_version_marker() {
let first_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").unwrap();
-17
View File
@@ -46,7 +46,6 @@ use crate::bucket::metadata_sys::BucketMetadataSys;
use crate::bucket::replication::{DynReplicationPool, ReplicationStats};
use crate::disk::DiskStore;
use crate::layout::endpoints::{EndpointServerPools, SetupType};
use crate::object_api::ObjectEncryptionResolver;
use crate::services::event_notification::EventNotifier;
use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
@@ -160,8 +159,6 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
#[cfg(test)]
@@ -200,7 +197,6 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
#[cfg(test)]
@@ -213,19 +209,6 @@ impl InstanceContext {
self.lock_manager.clone()
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
resolver: Arc<dyn ObjectEncryptionResolver>,
) -> Result<(), Arc<dyn ObjectEncryptionResolver>> {
self.object_encryption_resolver.set(resolver)
}
/// Return the configured object-encryption resolver, if startup installed one.
pub fn object_encryption_resolver(&self) -> Option<&dyn ObjectEncryptionResolver> {
self.object_encryption_resolver.get().map(Arc::as_ref)
}
/// Set this instance's S3 region.
///
/// Write-once: panics on a second write, preserving the startup fail-fast
+9
View File
@@ -46,6 +46,7 @@ use crate::{
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service};
use rustfs_lock::client::LockClient;
use s3s::dto::BucketLifecycleConfiguration;
use s3s::region::Region;
@@ -104,6 +105,10 @@ pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_
global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
}
pub(crate) async fn object_encryption_service() -> Option<Arc<ObjectEncryptionService>> {
get_global_encryption_service().await
}
pub fn object_store_handle() -> Option<Arc<ECStore>> {
resolve_object_store_handle()
}
@@ -202,6 +207,10 @@ pub(crate) async fn ensure_boot_time() {
GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
}
pub(crate) async fn scanner_init_time() -> Option<chrono::DateTime<chrono::Utc>> {
rustfs_common::get_global_init_time().await
}
pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option<u64> {
if is_erasure_sd().await {
None
@@ -71,7 +71,6 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
MadminScannerMetrics {
collected_at: metrics.collected_at,
current_cycle: metrics.current_cycle,
current_cycle_active: Some(metrics.current_cycle_active),
current_started: metrics.current_started,
cycles_completed_at: metrics.cycles_completed_at,
ongoing_buckets: metrics.ongoing_buckets,
@@ -399,7 +398,10 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
if types.contains(&MetricType::SCANNER) {
debug!("start get scanner metrics");
let metrics = global_metrics().report().await;
let mut metrics = global_metrics().report().await;
if let Some(init_time) = runtime_sources::scanner_init_time().await {
metrics.current_started = init_time;
}
real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics));
}
@@ -538,9 +540,7 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
#[cfg(test)]
mod test {
use super::*;
use rustfs_common::metrics::CurrentCycle;
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use serial_test::serial;
use std::time::Duration;
#[test]
@@ -588,10 +588,7 @@ mod test {
#[test]
fn scanner_metrics_mapping_preserves_partial_source_status() {
let current_started = Utc::now() - chrono::Duration::seconds(5);
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
current_cycle_active: true,
current_started,
last_cycle_partial_source: "usage".to_string(),
last_cycle_partial_source_code: 1,
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
@@ -601,8 +598,6 @@ mod test {
..Default::default()
});
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_started, current_started);
assert_eq!(scanner.last_cycle_partial_source, "usage");
assert_eq!(scanner.last_cycle_partial_source_code, 1);
let usage = scanner
@@ -613,39 +608,6 @@ mod test {
assert_eq!(usage.cycles, 2);
}
#[tokio::test]
#[serial]
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
let previous_init_time = *rustfs_common::globals::GLOBAL_INIT_TIME.read().await;
let previous_cycle = global_metrics().get_cycle().await;
let init_time = Utc::now() - chrono::Duration::hours(1);
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = Some(init_time);
let cycle = CurrentCycle {
current: 0,
next: 1,
started: cycle_started,
..Default::default()
};
let cycle_start = global_metrics().start_scan_cycle_work_with_cycle(cycle).await;
let realtime = collect_local_metrics(MetricType::SCANNER, &CollectMetricsOpts::default()).await;
global_metrics()
.finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
.await;
global_metrics().set_cycle(previous_cycle).await;
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
let mut aggregated = RealtimeMetrics::default();
aggregated.merge(decoded);
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_started, cycle_started);
}
#[test]
fn scanner_metrics_mapping_preserves_pacing_pressure() {
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
+3 -389
View File
@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_madmin::net::NetInfo;
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
use rustfs_utils::XHost;
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
use std::collections::{HashMap, hash_map::DefaultHasher};
use std::future::Future;
use std::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant, SystemTime};
use std::time::{Duration, SystemTime};
use tokio::time::{sleep, timeout};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
@@ -47,9 +47,6 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
/// Cached result from the last successful admin call to a peer.
struct PeerAdminCache {
@@ -94,180 +91,6 @@ lazy_static! {
pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
}
#[derive(Clone)]
struct RemoteVersionStateFleetProof {
topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>,
expires_at: Instant,
}
impl RemoteVersionStateFleetProof {
fn token(&self) -> RemoteVersionStateFleetProofToken {
RemoteVersionStateFleetProofToken {
topology_fingerprint: self.topology_fingerprint.clone(),
peer_epochs: self.peer_epochs.clone(),
}
}
}
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct RemoteVersionStateFleetProofToken {
topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>,
}
#[derive(Default)]
struct RemoteVersionStateFleetProofState {
proof: Option<RemoteVersionStateFleetProof>,
topology_conflict: bool,
}
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
}
fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
}
fn replace_remote_version_state_fleet_proof_in(
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
proof: Option<RemoteVersionStateFleetProof>,
) {
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
}
fn publish_remote_version_state_probe_result(
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
topology_fingerprint: &str,
result: Result<BTreeMap<String, Uuid>>,
observed_at: Instant,
) -> Option<Error> {
match result {
Ok(peer_epochs) => {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
let peer_epochs = state
.proof
.as_ref()
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
.map(|proof| Arc::clone(&proof.peer_epochs))
.unwrap_or_else(|| Arc::new(peer_epochs));
state.proof = Some(RemoteVersionStateFleetProof {
topology_fingerprint: topology_fingerprint.to_string(),
peer_epochs,
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
});
None
}
Err(err) => {
replace_remote_version_state_fleet_proof_in(slot, None);
Some(err)
}
}
}
pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
let state = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
}
fn acquire_remote_version_state_fleet_proof_from(
state: &RemoteVersionStateFleetProofState,
expected_topology: &str,
now: Instant,
) -> Option<RemoteVersionStateFleetProofToken> {
if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
return None;
}
state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
}
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
return false;
};
let state = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.topology_conflict {
return false;
}
state.proof.as_ref().is_some_and(|current| {
current.topology_fingerprint == *expected_topology
&& current.topology_fingerprint == proof.topology_fingerprint
&& Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
&& Instant::now() < current.expires_at
})
}
fn remote_version_state_fleet_proof_valid_at(
proof: Option<&RemoteVersionStateFleetProof>,
expected_topology: &str,
now: Instant,
) -> bool {
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
}
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
return Err(Error::other("remote version state capability peer identity is invalid"));
}
Ok(())
}
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
let mut state = remote_version_state_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.topology_conflict = true;
state.proof = None;
}
return;
}
tokio::spawn(async move {
loop {
let result = match get_global_notification_sys() {
Some(notification_sys) => {
match timeout(
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
)
.await
{
Ok(result) => result,
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
}
}
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
};
let topology_conflict = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.topology_conflict;
if topology_conflict {
replace_remote_version_state_fleet_proof(None);
} else if let Some(err) = publish_remote_version_state_probe_result(
remote_version_state_fleet_proof_slot(),
&topology_fingerprint,
result,
Instant::now(),
) {
debug!(error = %err, "remote version state fleet capability probe failed closed");
}
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
}
});
}
pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
let _ = GLOBAL_NOTIFICATION_SYS
.set(Arc::new(NotificationSys::new(eps).await))
@@ -292,17 +115,7 @@ pub struct NotificationSys {
impl NotificationSys {
pub async fn new(eps: EndpointServerPools) -> Self {
let expected_remote_hosts = eps
.peer_grid_host_slots_sorted()
.into_iter()
.filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
.collect::<Vec<_>>();
let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
let peer_topology_hosts = if peer_topology_hosts.is_empty() {
expected_remote_hosts
} else {
peer_topology_hosts
};
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
Self {
peer_clients,
@@ -312,24 +125,6 @@ impl NotificationSys {
tier_config_reload_workers: Default::default(),
}
}
async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
if self.peer_clients.len() != self.peer_topology_hosts.len() {
return Err(Error::other("remote version state capability fleet membership is incomplete"));
}
let probes = self.peer_clients.iter().map(|client| async {
let client = client
.as_ref()
.ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
client.probe_remote_version_state(topology_fingerprint.to_string()).await
});
let mut peer_epochs = BTreeMap::new();
for result in join_all(probes).await {
let (peer, epoch) = result?;
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
}
Ok(peer_epochs)
}
}
pub struct NotificationPeerErr {
@@ -1549,11 +1344,8 @@ async fn run_tier_config_reload_worker<F, Fut>(
}
TierConfigReloadFinish::Pending => retry_attempt = 0,
},
TierConfigReloadOutcome::Terminal(err) => match sys.finish_tier_config_reload_worker(&host) {
TierConfigReloadOutcome::Terminal(_) => match sys.finish_tier_config_reload_worker(&host) {
TierConfigReloadFinish::Completed => {
// This peer keeps the previous tier configuration for good, so record
// why. Dropping the error here hides the only evidence of a divergent
// node behind an outcome label that cannot be acted on.
warn!(
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
component = LOG_COMPONENT_ECSTORE,
@@ -1561,7 +1353,6 @@ async fn run_tier_config_reload_worker<F, Fut>(
action = "reload_transition_tier_config",
host,
outcome = "terminal",
error = ?err,
"tier configuration reload stopped after a terminal outcome"
);
return;
@@ -2095,183 +1886,6 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
mod tests {
use super::*;
#[test]
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
let now = Instant::now();
let mut peer_epochs = BTreeMap::new();
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(peer_epochs),
expires_at: now + Duration::from_secs(1),
};
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
}
#[test]
fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
let mut peer_epochs = BTreeMap::new();
assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
assert!(peer_epochs.is_empty());
}
#[test]
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
let now = Instant::now();
let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
};
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
}
#[test]
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
let now = Instant::now();
let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: now + Duration::from_secs(1),
};
let captured = proof.token();
let restarted = RemoteVersionStateFleetProof {
topology_fingerprint: proof.topology_fingerprint.clone(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: proof.expires_at,
};
assert!(captured != restarted.token());
}
#[test]
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
let now = Instant::now();
let epoch = Uuid::new_v4();
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
let original = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("successful probe should publish proof")
.token();
assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
);
let renewed = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("renewal should retain proof")
.token();
assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
.is_none()
);
let replaced = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("restarted peer should publish a new proof")
.token();
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
}
#[test]
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
let now = Instant::now();
let mut state = RemoteVersionStateFleetProofState {
proof: Some(RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
}),
topology_conflict: false,
};
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
state.topology_conflict = true;
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
}
#[test]
fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
let epoch = Uuid::new_v4();
let mut peer_epochs = BTreeMap::new();
insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
.expect("first member should be admitted");
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
}
#[test]
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
let now = Instant::now();
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
);
assert!(slot.read().expect("proof slot should not poison").proof.is_none());
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
}
#[tokio::test]
async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
let notification_sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: vec![None, None],
peer_topology_hosts: vec!["peer-a".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
tier_config_reload_workers: Default::default(),
};
let err = notification_sys
.probe_remote_version_state_fleet("topology-a")
.await
.expect_err("an unreachable configured member must fail the fleet proof");
assert!(err.to_string().contains("unreachable"));
}
#[tokio::test]
async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
let notification_sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: vec![None],
peer_topology_hosts: vec!["peer-a".to_string()],
peer_admin_caches: Vec::new(),
tier_config_reload_workers: Default::default(),
};
let err = notification_sys
.probe_remote_version_state_fleet("topology-a")
.await
.expect_err("a missing configured member slot must fail the fleet proof");
assert!(err.to_string().contains("incomplete"));
}
fn build_props(endpoint: &str) -> ServerProperties {
ServerProperties {
endpoint: endpoint.to_string(),
-39
View File
@@ -236,7 +236,6 @@ struct TierPublishTransition {
struct PreparedTierDriver {
tier_name: String,
tier_config: TierConfig,
config_fingerprint: TierDriverFingerprint,
backend_identity: TierDestinationId,
exact_get_delete: bool,
@@ -1343,7 +1342,6 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
struct TierDriverGeneration {
tier_name: Arc<str>,
tier_config: TierConfig,
generation: DriverRevision,
// Process-local only: this may reflect credential changes and must never be persisted or logged.
config_fingerprint: TierDriverFingerprint,
@@ -1351,9 +1349,6 @@ struct TierDriverGeneration {
backend_identity: TierDestinationId,
exact_get_delete: bool,
driver: SharedWarmBackend,
reconciler: tokio::sync::OnceCell<
Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
>,
accepting: AtomicBool,
active_leases: AtomicUsize,
drained: tokio::sync::Notify,
@@ -1478,35 +1473,6 @@ impl TierOperationLease {
self.inner.backend_identity
}
pub(crate) async fn probe_transition_candidate_for(
&self,
object: &str,
transaction_id: uuid::Uuid,
) -> io::Result<TransitionCandidateProbe> {
let Some(reconciler) = self
.inner
.reconciler
.get_or_try_init(|| async {
crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
.await
.map(|reconciler| reconciler.map(Arc::from))
})
.await
.map_err(|err| io::Error::other(err.message))?
else {
return self.inner.driver.probe_transition_candidate(object).await;
};
reconciler
.probe_transition_candidate_for(
object,
crate::services::tier::warm_backend::TransitionCandidateIdentity {
transaction_id,
destination_id: self.backend_identity(),
},
)
.await
}
pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
self.inner.driver.validate_remote_version_id(remote_version_id)?;
if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
@@ -2867,7 +2833,6 @@ impl TierConfigMgr {
let exact_get_delete = tier_exact_get_delete(config);
Some(PreparedTierDriver {
tier_name: tier_name.to_string(),
tier_config: config.clone(),
config_fingerprint,
backend_identity,
exact_get_delete,
@@ -2896,13 +2861,11 @@ impl TierConfigMgr {
})?;
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(prepared.tier_name.as_str()),
tier_config: prepared.tier_config.clone(),
generation,
config_fingerprint: prepared.config_fingerprint,
backend_identity: prepared.backend_identity,
exact_get_delete: prepared.exact_get_delete,
driver: prepared.driver.clone(),
reconciler: tokio::sync::OnceCell::new(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
@@ -3646,13 +3609,11 @@ impl TierConfigMgr {
let driver: SharedWarmBackend = Arc::from(driver);
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(tier_name),
tier_config: config.clone(),
generation,
config_fingerprint,
backend_identity,
exact_get_delete,
driver: driver.clone(),
reconciler: tokio::sync::OnceCell::new(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
@@ -73,21 +73,6 @@ pub enum TransitionCandidateProbe {
Unsupported,
}
#[derive(Clone, Copy)]
pub(crate) struct TransitionCandidateIdentity {
pub transaction_id: uuid::Uuid,
pub destination_id: [u8; 32],
}
#[async_trait::async_trait]
pub(crate) trait TransitionCandidateReconciler {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error>;
}
#[async_trait::async_trait]
pub trait WarmBackend {
async fn validate(&self) -> Result<(), std::io::Error> {
@@ -204,20 +189,6 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
metadata.remove(key);
}
for suffix in [
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
] {
for key in [
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
] {
if let Some(value) = metadata.remove(&key) {
metadata.insert(format!("x-amz-meta-{key}"), value);
}
}
}
opts.user_metadata = metadata;
opts
}
@@ -475,51 +446,6 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
Ok(d)
}
pub(crate) async fn new_transition_candidate_reconciler(
tier: &TierConfig,
) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
TierType::S3 => Box::new(
WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
TierType::MinIO => Box::new(
WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
TierType::RustFS => Box::new(
WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
TierType::R2 => Box::new(
WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
_ => return Ok(None),
};
Ok(Some(reconciler))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -849,37 +775,6 @@ mod tests {
assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
}
#[test]
fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
let mut metadata = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
"5a".repeat(32),
);
let opts = build_transition_put_options("COLD".to_string(), metadata);
for suffix in [
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
] {
assert!(opts.user_metadata.contains_key(&format!(
"x-amz-meta-{}",
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
)));
assert!(opts.user_metadata.contains_key(&format!(
"x-amz-meta-{}{suffix}",
rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
)));
}
}
#[test]
fn build_transition_put_options_requests_no_checksum_and_content_md5() {
// Regression for rustfs/rustfs#4811: transition uploads must leave the
@@ -19,7 +19,6 @@
#![allow(clippy::all)]
use std::collections::HashMap;
use std::io::{Error, ErrorKind};
use std::sync::Arc;
use bytes::Bytes;
@@ -46,19 +45,6 @@ const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
if remote_version.is_empty() {
return Ok(None);
}
let generation = remote_version
.parse::<i64>()
.map_err(|_| Error::new(ErrorKind::InvalidData, "GCS remote version is not a valid generation"))?;
if generation <= 0 {
return Err(Error::new(ErrorKind::InvalidData, "GCS remote version generation must be positive"));
}
Ok(Some(generation))
}
pub struct WarmBackendGCS {
pub client: Arc<Storage>,
pub control: Arc<StorageControl>,
@@ -119,10 +105,6 @@ impl WarmBackendGCS {
#[async_trait::async_trait]
impl WarmBackend for WarmBackendGCS {
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
parse_generation(remote_version_id).map(|_| ())
}
async fn put_with_meta(
&self,
object: &str,
@@ -153,9 +135,6 @@ impl WarmBackend for WarmBackendGCS {
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
}
// Honor the requested byte range so Range GETs on tiered objects return the exact
// interval instead of the whole object (matches the s3/s3sdk/rustfs warm backends).
@@ -185,15 +164,13 @@ impl WarmBackend for WarmBackendGCS {
// gRPC v2 DeleteObject requires the bucket in resource-name form. Without this the
// deleted tiered object was never removed from GCS (empty impl returned Ok), leaking
// remote data forever.
let mut req = self
.control
self.control
.delete_object()
.set_bucket(format!("projects/_/buckets/{}", self.bucket))
.set_object(self.get_dest(object));
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
}
req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
.set_object(self.get_dest(object))
.send()
.await
.map_err(|e| std::io::Error::other(e.to_string()))?;
Ok(())
}
@@ -214,30 +191,6 @@ impl WarmBackend for WarmBackendGCS {
}
}
#[cfg(test)]
mod tests {
use super::parse_generation;
use std::io::ErrorKind;
#[test]
fn generation_parser_preserves_exact_numeric_versions() {
assert_eq!(parse_generation("").expect("empty generation means no version condition"), None);
assert_eq!(parse_generation("1").expect("minimum generation should parse"), Some(1));
assert_eq!(
parse_generation(&i64::MAX.to_string()).expect("maximum generation should parse"),
Some(i64::MAX)
);
}
#[test]
fn generation_parser_rejects_unknown_or_non_positive_versions() {
for value in ["unknown", "1.0", "-1", "0", "9223372036854775808"] {
let err = parse_generation(value).expect_err("unknown generation must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData, "{value}");
}
}
}
/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
if err == nil {
return nil
@@ -135,20 +135,6 @@ impl WarmBackend for WarmBackendMinIO {
}
}
#[async_trait::async_trait]
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
&self.0, object, identity,
)
.await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
@@ -135,20 +135,6 @@ impl WarmBackend for WarmBackendR2 {
}
}
#[async_trait::async_trait]
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
&self.0, object, identity,
)
.await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
@@ -132,20 +132,6 @@ impl WarmBackend for WarmBackendRustFS {
}
}
#[async_trait::async_trait]
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
&self.0, object, identity,
)
.await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
@@ -29,7 +29,6 @@ use crate::client::{
api_remove::{RemoveObjectOptions, RemoveObjectResult},
api_s3_datatypes::ListVersionsResult,
credentials::{Credentials, SignatureType, Static, Value},
provider_versions::validate_remote_version_id,
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
transition_api::{ReadCloser, ReaderImpl},
};
@@ -37,10 +36,7 @@ use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::services::tier::{
tier_config::TierS3,
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options,
},
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
};
use http::HeaderMap;
use rustfs_utils::egress::validate_outbound_url;
@@ -193,188 +189,44 @@ impl WarmBackendS3 {
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
}
async fn probe_transition_candidate_versions(
&self,
object: &str,
bucket_versioning: RemoteBucketVersioning,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let remote_object = self.get_dest(object);
async fn list_transition_candidate_versions(&self, object: &str) -> Result<ListVersionsResult, std::io::Error> {
let mut opts = ListObjectsOptions::default();
opts.set("prefix", &remote_object);
opts.set("max-keys", "1000");
let mut key_marker = String::new();
let mut version_id_marker = String::new();
let mut candidates = TransitionCandidateVersions::default();
loop {
let versions = self
.client
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
.await?;
candidates.extend(&remote_object, &versions);
if candidates.is_ambiguous() {
return Ok(TransitionCandidateProbe::Ambiguous);
}
if !versions.is_truncated {
return classify_transition_candidates(candidates, bucket_versioning);
}
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
}
}
async fn probe_transition_candidate_identity(
&self,
object: &str,
identity: TransitionCandidateIdentity,
bucket_versioning: RemoteBucketVersioning,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let remote_object = self.get_dest(object);
let mut opts = ListObjectsOptions::default();
opts.set("prefix", &remote_object);
opts.set("max-keys", "1000");
let mut key_marker = String::new();
let mut version_id_marker = String::new();
let mut matched_version = None;
let mut saw_unproven_candidate = false;
loop {
let versions = self
.client
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
.await?;
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
let mut stat_opts = GetObjectOptions::default();
stat_opts.version_id.clone_from(&version.version_id);
let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
let mut metadata = info.user_metadata;
for (name, value) in &info.metadata {
if (name
.as_str()
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|| name
.as_str()
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
&& let Ok(value) = value.to_str()
{
metadata.insert(name.as_str().to_string(), value.to_string());
}
}
if transition_candidate_metadata_matches(&metadata, identity)? {
if matched_version.is_some() {
return Ok(TransitionCandidateProbe::Ambiguous);
}
matched_version = Some(version.version_id.clone());
} else {
saw_unproven_candidate = true;
}
}
if !versions.is_truncated {
if matched_version.is_none() && saw_unproven_candidate {
return Ok(TransitionCandidateProbe::Unsupported);
}
let candidates = TransitionCandidateVersions {
version_id: matched_version,
ambiguous: false,
};
return classify_transition_candidates(candidates, bucket_versioning);
}
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
}
opts.set("prefix", &self.get_dest(object));
opts.set("max-keys", "2");
self.client.list_object_versions_query(&self.bucket, &opts, "", "", "").await
}
}
fn transition_candidate_metadata_matches(
metadata: &HashMap<String, String>,
identity: TransitionCandidateIdentity,
) -> Result<bool, std::io::Error> {
use rustfs_utils::http::metadata_compat::{
SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
};
let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
if transaction_id.is_none() || destination_id.is_none() {
if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
|| contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
{
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"transition candidate identity metadata is empty or conflicting",
));
}
return Ok(false);
}
let expected_transaction_id = identity.transaction_id.to_string();
let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
}
fn classify_transition_candidates(
candidates: TransitionCandidateVersions,
fn classify_transition_candidate_versions(
remote_object: &str,
bucket_versioning: RemoteBucketVersioning,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let probe = candidates.classify(bucket_versioning);
if let TransitionCandidateProbe::VersionedPresent(version_id) = &probe {
validate_remote_version_id(version_id)?;
}
Ok(probe)
}
fn advance_version_markers(
key_marker: &mut String,
version_id_marker: &mut String,
versions: &ListVersionsResult,
) -> Result<(), std::io::Error> {
let next_markers = (&versions.next_key_marker, &versions.next_version_id_marker);
if next_markers == (&*key_marker, &*version_id_marker) {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"ListObjectVersions pagination markers did not advance",
));
}
key_marker.clone_from(&versions.next_key_marker);
version_id_marker.clone_from(&versions.next_version_id_marker);
Ok(())
}
#[derive(Default)]
struct TransitionCandidateVersions {
version_id: Option<String>,
ambiguous: bool,
}
impl TransitionCandidateVersions {
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
if self.version_id.is_some() {
self.ambiguous = true;
return;
}
self.version_id = Some(version.version_id.clone());
}
) -> TransitionCandidateProbe {
if versions.is_truncated {
return TransitionCandidateProbe::Ambiguous;
}
fn is_ambiguous(&self) -> bool {
self.ambiguous
if versions.delete_markers.iter().any(|marker| marker.key == remote_object) {
return TransitionCandidateProbe::Ambiguous;
}
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
if self.ambiguous {
return TransitionCandidateProbe::Ambiguous;
}
let Some(version_id) = self.version_id else {
return TransitionCandidateProbe::Missing;
};
let mut exact_versions = versions.versions.iter().filter(|version| version.key == remote_object);
let Some(version) = exact_versions.next() else {
return TransitionCandidateProbe::Missing;
};
if exact_versions.next().is_some() {
return TransitionCandidateProbe::Ambiguous;
}
match bucket_versioning {
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
RemoteBucketVersioning::Suspended if version_id == "null" => TransitionCandidateProbe::VersionedPresent(version_id),
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version_id.is_empty() => {
TransitionCandidateProbe::VersionedPresent(version_id)
}
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
match bucket_versioning {
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
RemoteBucketVersioning::Suspended if version.version_id == "null" => {
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
}
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version.version_id.is_empty() => {
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
}
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
}
}
@@ -423,161 +275,72 @@ mod tests {
}
}
fn classify_pages(bucket_versioning: RemoteBucketVersioning, pages: &[ListVersionsResult]) -> TransitionCandidateProbe {
let mut candidates = TransitionCandidateVersions::default();
for page in pages {
candidates.extend("archive/object", page);
}
candidates.classify(bucket_versioning)
}
fn candidate_identity() -> TransitionCandidateIdentity {
TransitionCandidateIdentity {
transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
destination_id: [0x5a; 32],
}
}
fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
let mut metadata = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
identity.transaction_id.to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
rustfs_utils::crypto::hex(identity.destination_id),
);
metadata
}
#[test]
fn transition_candidate_identity_requires_exact_compatible_metadata() {
let identity = candidate_identity();
let metadata = candidate_metadata(identity);
assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
let mut adjacent = metadata;
rustfs_utils::http::metadata_compat::insert_str(
&mut adjacent,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
.unwrap()
.to_string(),
);
assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
}
#[test]
fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
let identity = candidate_identity();
let mut metadata = candidate_metadata(identity);
metadata.insert(
rustfs_utils::http::metadata_compat::internal_key_rustfs(
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
),
uuid::Uuid::new_v4().to_string(),
);
assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
}
#[test]
fn transition_candidate_probe_classifier_is_fail_closed() {
assert_eq!(
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[], &[], false)],),
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Disabled,
&list_versions(&[], &[], false),
),
TransitionCandidateProbe::Missing
);
assert_eq!(
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[("archive/object", "")], &[], false)],),
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Disabled,
&list_versions(&[("archive/object", "")], &[], false),
),
TransitionCandidateProbe::UnversionedPresent
);
assert_eq!(
classify_pages(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Enabled,
&[list_versions(&[("archive/object", "version-a")], &[], false)],
&list_versions(&[("archive/object", "version-a")], &[], false),
),
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
);
assert_eq!(
classify_pages(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Suspended,
&[list_versions(&[("archive/object", "null")], &[], false)],
&list_versions(&[("archive/object", "null")], &[], false),
),
TransitionCandidateProbe::VersionedPresent("null".to_string())
);
assert_eq!(
classify_pages(RemoteBucketVersioning::Enabled, &[list_versions(&[("archive/object", "")], &[], false)],),
TransitionCandidateProbe::Ambiguous
);
assert_eq!(
classify_pages(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Enabled,
&[list_versions(
&[("archive/object", "version-a"), ("archive/object", "version-b")],
&[],
false,
)],
&list_versions(&[("archive/object", "")], &[], false),
),
TransitionCandidateProbe::Ambiguous
);
}
#[test]
fn transition_candidate_probe_reconciles_all_pages_and_ignores_delete_markers() {
assert_eq!(
classify_pages(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Enabled,
&[
list_versions(&[], &[("archive/object", "marker-a")], true),
list_versions(&[("archive/object", "version-a"), ("archive/object-adjacent", "unrelated"),], &[], false,),
],
),
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
);
assert_eq!(
classify_pages(
RemoteBucketVersioning::Enabled,
&[
list_versions(&[("archive/object", "version-a")], &[], true),
list_versions(&[("archive/object", "version-b")], &[], false),
],
&list_versions(&[("archive/object", "version-a"), ("archive/object", "version-b")], &[], false),
),
TransitionCandidateProbe::Ambiguous
);
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Enabled,
&list_versions(&[("archive/object", "version-a")], &[("archive/object", "marker-a")], false),
),
TransitionCandidateProbe::Ambiguous
);
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Enabled,
&list_versions(&[("archive/object", "version-a")], &[], true),
),
TransitionCandidateProbe::Ambiguous
);
}
#[test]
fn transition_candidate_pagination_advances_both_markers() {
let mut key_marker = "old-key".to_string();
let mut version_id_marker = "old-version".to_string();
let page = ListVersionsResult {
next_key_marker: "next-key".to_string(),
next_version_id_marker: "next-version".to_string(),
..Default::default()
};
advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
.expect("new ListObjectVersions markers should advance pagination");
assert_eq!(key_marker, "next-key");
assert_eq!(version_id_marker, "next-version");
let err = advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
.expect_err("repeated ListObjectVersions markers must fail closed");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
#[test]
fn transition_candidate_probe_rejects_untrusted_version_ids() {
let mut candidates = TransitionCandidateVersions::default();
candidates.extend("archive/object", &list_versions(&[("archive/object", "version\ninjection")], &[], false));
let err = classify_transition_candidates(candidates, RemoteBucketVersioning::Enabled)
.expect_err("control characters in listed version IDs must fail closed");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
#[test]
@@ -634,7 +397,12 @@ impl WarmBackend for WarmBackendS3 {
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
let bucket_versioning = self.remote_bucket_versioning().await?;
self.probe_transition_candidate_versions(object, bucket_versioning).await
let versions = self.list_transition_candidate_versions(object).await?;
Ok(classify_transition_candidate_versions(
&self.get_dest(object),
bucket_versioning,
&versions,
))
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
@@ -646,16 +414,3 @@ impl WarmBackend for WarmBackendS3 {
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
}
}
#[async_trait::async_trait]
impl TransitionCandidateReconciler for WarmBackendS3 {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let bucket_versioning = self.remote_bucket_versioning().await?;
self.probe_transition_candidate_identity(object, identity, bucket_versioning)
.await
}
}
@@ -46,7 +46,6 @@ use crate::diagnostics::get::{
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
use crate::disk::{
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
PartTransactionAction, part_transaction_path,
@@ -3953,9 +3952,9 @@ impl SetDisks {
/// * The set of referenced data dirs is the UNION of `get_data_dirs()` across
/// every online disk's `xl.meta`, so a dir named by *any* replica is kept.
/// * If a disk holds the object directory but its `xl.meta` is missing or
/// unparsable, the object is treated as degraded and unmarked data dirs are
/// never removed. A data dir carrying a committed delete-transaction marker
/// remains reclaimable after a downgrade/re-upgrade cleanup interruption.
/// unparsable, the object is treated as degraded and NOTHING is removed —
/// the unreadable copy could be the only one naming a live data dir, and a
/// heal must run first.
/// * Only subdirectories whose names parse as a UUID are ever considered;
/// removal is non-recursive-safe via a recursive delete of the full stray
/// data-dir path only.
@@ -3970,7 +3969,7 @@ impl SetDisks {
// physical UUID subdirectories present on each disk. Abort on any degraded
// copy so a healable object is never stripped of a referenced data dir.
let mut referenced: HashSet<Uuid> = HashSet::new();
let mut per_disk_dirs: Vec<(usize, Vec<(Uuid, bool)>)> = Vec::new();
let mut per_disk_dirs: Vec<(usize, Vec<Uuid>)> = Vec::new();
let mut healthy_metas = 0usize;
for (i, disk) in disks.iter().enumerate() {
@@ -4006,22 +4005,6 @@ impl SetDisks {
// to the orphan-dir / dangling-object heal paths.
continue;
}
let mut committed = Vec::with_capacity(physical.len());
for dir in physical {
let data_dir = format!("{object}/{dir}");
let committed_delete = disk.list_dir("", bucket, &data_dir, 0).await.is_ok_and(|entries| {
entries.iter().any(|entry| {
entry
.strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
.is_some_and(|transaction| Uuid::parse_str(transaction).is_ok())
})
});
committed.push((dir, committed_delete));
}
if committed.iter().all(|(_, committed_delete)| *committed_delete) {
per_disk_dirs.push((i, committed));
continue;
}
warn!(
target: "rustfs_ecstore::set_disk",
bucket, object,
@@ -4058,16 +4041,22 @@ impl SetDisks {
healthy_metas += 1;
if !physical.is_empty() {
per_disk_dirs.push((i, physical.into_iter().map(|dir| (dir, false)).collect()));
per_disk_dirs.push((i, physical));
}
}
// No healthy metadata anywhere: this is not a live object, so surplus dirs
// (if any) belong to the dangling-object heal path, not here.
if healthy_metas == 0 {
return Ok(0);
}
// Phase 2: delete every physical data dir not referenced by the union.
let mut removed = 0usize;
for (i, physical) in per_disk_dirs {
let Some(disk) = disks[i].as_ref() else { continue };
for (dir, committed_delete) in physical {
if referenced.contains(&dir) || (healthy_metas == 0 && !committed_delete) {
for dir in physical {
if referenced.contains(&dir) {
continue;
}
let stray = format!("{object}/{dir}");
+4 -2
View File
@@ -505,7 +505,9 @@ impl SetDisks {
}
fn file_info_has_encryption_metadata(meta: &FileInfo) -> bool {
meta.metadata.keys().any(|name| http::is_object_encryption_marker(name))
meta.metadata
.keys()
.any(|name| http::is_encryption_metadata_key(name) || http::is_sse_header(name))
}
fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
@@ -553,7 +555,7 @@ impl SetDisks {
}
}
pub(super) fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
let mut hasher = Sha256::new();
Self::update_file_info_quorum_hash(&mut hasher, meta);
let digest = hasher.finalize();
File diff suppressed because it is too large Load Diff
+136 -650
View File
@@ -92,69 +92,6 @@ struct PartFailureSummary {
bitrot_failure: bool,
}
#[derive(Clone)]
struct RecoverableMetaCandidate {
identity: [u8; 32],
file_info: FileInfo,
data_count: usize,
local_payload: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum DanglingDeleteSafety {
UnsafeToDelete,
NoRecoverableCandidate,
}
#[cfg(test)]
struct DanglingCheckPartsFailure {
key: DanglingCheckPartsFailureKey,
}
#[cfg(test)]
type DanglingCheckPartsFailureKey = (String, String, usize);
#[cfg(test)]
type DanglingCheckPartsFailures = HashMap<DanglingCheckPartsFailureKey, DiskError>;
#[cfg(test)]
fn dangling_check_parts_failures() -> &'static std::sync::Mutex<DanglingCheckPartsFailures> {
static FAILURES: std::sync::OnceLock<std::sync::Mutex<DanglingCheckPartsFailures>> = std::sync::OnceLock::new();
FAILURES.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
}
#[cfg(test)]
impl DanglingCheckPartsFailure {
fn install(bucket: &str, object: &str, disk_index: usize, error: DiskError) -> Self {
let key = (bucket.to_string(), object.to_string(), disk_index);
let previous = dangling_check_parts_failures()
.lock()
.expect("dangling check-parts failure registry should not poison")
.insert(key.clone(), error);
assert!(previous.is_none(), "dangling check-parts failure already installed");
Self { key }
}
}
#[cfg(test)]
impl Drop for DanglingCheckPartsFailure {
fn drop(&mut self) {
dangling_check_parts_failures()
.lock()
.expect("dangling check-parts failure registry should not poison")
.remove(&self.key);
}
}
#[cfg(test)]
fn injected_dangling_check_parts_error(bucket: &str, object: &str, disk_index: usize) -> Option<DiskError> {
dangling_check_parts_failures()
.lock()
.expect("dangling check-parts failure registry should not poison")
.get(&(bucket.to_string(), object.to_string(), disk_index))
.cloned()
}
fn first_unhealthy_part_summary(
data_errs_by_part: &HashMap<usize, Vec<usize>>,
parts: &[ObjectPartInfo],
@@ -188,17 +125,39 @@ impl SetDisks {
version_id: &str,
opts: &HealOpts,
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, true)).await
// `allow_meta_regen` is true on the first pass: a version whose data shards
// physically survive (>= data_blocks) but whose xl.meta fell below
// read-quorum is RESCUED (missing xl.meta regenerated) rather than
// dangling-deleted. The re-drive after a rescue sets it false so the
// regeneration can happen at most once (no unbounded recursion).
Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, true)).await
}
/// Best-effort orphan-data-dir reclaim for an object that is healthy on this
/// set. Wraps [`Self::reclaim_orphan_data_dirs`] with the shared logging so
/// both `heal_object` exits — the already-healthy early return and the
/// post-heal tail — reclaim identically. Never fails the heal: delete errors
/// are logged and swallowed. Callers must gate this on `!opts.dry_run`.
async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
match self.reclaim_orphan_data_dirs(bucket, object).await {
Ok(removed) if removed > 0 => {
info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
}
Ok(_) => {}
Err(e) => {
warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
}
}
}
#[allow(clippy::too_many_lines)]
async fn heal_object_with_explicit_version_regen(
async fn heal_object_with_regen(
&self,
bucket: &str,
object: &str,
version_id: &str,
opts: &HealOpts,
allow_explicit_version_regen: bool,
allow_meta_regen: bool,
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
info!(?opts, "Starting heal_object");
@@ -395,6 +354,22 @@ impl SetDisks {
}
}
// DATA-SAFETY GUARD (backlog#920, decision 1): before any
// dangling delete, if the version's DATA shards physically
// survive on >= data_blocks disks it is RECONSTRUCTABLE.
// Regenerate the missing xl.meta from a surviving valid
// FileInfo and re-drive the heal instead of destroying a
// recoverable version. Torn writes (< data_blocks data
// shards) fall through to the existing dangling behavior.
if cannot_heal
&& allow_meta_regen
&& self
.try_regenerate_recoverable_meta(bucket, object, &parts_metadata, &errs, &disks)
.await?
{
return Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, false)).await;
}
if cannot_heal {
let total_disks = parts_metadata.len();
let healthy_count = total_disks.saturating_sub(disks_to_heal_count);
@@ -447,20 +422,6 @@ impl SetDisks {
);
}
// `disks_with_all_parts` normalizes conflicting entries
// in `parts_metadata` to defaults. Re-read only before
// destructive cleanup so the guard sees every original
// identity.
let (delete_guard_metadata, delete_guard_errs) =
Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?;
if self
.dangling_delete_safety(bucket, object, &delete_guard_metadata, &delete_guard_errs, &disks)
.await?
== DanglingDeleteSafety::UnsafeToDelete
{
return Ok((result, Some(cannot_heal_err)));
}
// Allow for dangling deletes, on versions that have DataDir missing etc.
// this would end up restoring the correct readable versions.
return match self
@@ -876,25 +837,17 @@ impl SetDisks {
}
}
Err(err) => {
if allow_explicit_version_regen
&& !version_id.is_empty()
// DATA-SAFETY GUARD (backlog#920, decision 1): meta quorum failed,
// but the version's DATA may still physically survive on enough
// disks (xl.meta lost on > parity disks while part files remain).
// Rescue it by regenerating the missing xl.meta and re-driving heal
// instead of dangling-deleting a reconstructable version.
if allow_meta_regen
&& self
.try_regenerate_explicit_version_meta(bucket, object, version_id, &parts_metadata, &errs, &disks)
.try_regenerate_recoverable_meta(bucket, object, &parts_metadata, &errs, &disks)
.await?
{
return Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, false)).await;
}
if self
.dangling_delete_safety(bucket, object, &parts_metadata, &errs, &disks)
.await?
== DanglingDeleteSafety::UnsafeToDelete
{
return Ok((
self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
.await,
Some(err),
));
return Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, false)).await;
}
let data_errs_by_part = HashMap::new();
@@ -930,226 +883,129 @@ impl SetDisks {
}
}
async fn try_regenerate_explicit_version_meta(
/// backlog#920 (decision 1): rescue a version that meta-quorum logic would
/// otherwise dangling-DELETE, when its DATA is still reconstructable.
///
/// Returns `Ok(true)` if the version was rescued (missing xl.meta regenerated
/// on at least one disk, so a re-driven heal can reconstruct it), `Ok(false)`
/// to fall through to the existing dangling-delete behavior.
///
/// Recoverability is computed by physically probing part files across ALL
/// disks in the set with `check_parts` — including disks whose xl.meta is
/// absent (a lost xl.meta does not lose the sibling `part.*` data). If at
/// least `data_blocks` disks hold every part of a surviving valid FileInfo,
/// the object is EC-reconstructable, so we regenerate that FileInfo's xl.meta
/// on every disk whose metadata is absent (via `write_metadata`, which merges
/// into any existing xl.meta). Delete markers, remote/transitioned versions,
/// and genuine torn writes (< `data_blocks` surviving data shards) are NOT
/// rescued — they keep the current dangling-delete-after-grace behavior, so no
/// regression on those paths.
async fn try_regenerate_recoverable_meta(
&self,
bucket: &str,
object: &str,
version_id: &str,
parts_metadata: &[FileInfo],
errs: &[Option<DiskError>],
disks: &[Option<DiskStore>],
) -> disk::error::Result<bool> {
let Ok(version_id) = Uuid::parse_str(version_id) else {
// A surviving valid, non-deleted, non-remote data FileInfo to rebuild from.
let Some(surviving) = parts_metadata
.iter()
.find(|fi| fi.has_valid_erasure_geometry() && !fi.deleted && !fi.is_remote())
.cloned()
else {
return Ok(false);
};
let candidates = parts_metadata
.iter()
.zip(errs.iter())
.filter_map(|(file_info, err)| {
(err.is_none()
&& file_info_is_valid_for_metadata(file_info)
&& file_info.version_id == Some(version_id)
&& file_info.has_valid_erasure_geometry()
&& !file_info.deleted
&& !file_info.is_remote()
&& file_info.data_dir.is_some()
&& !file_info.parts.is_empty()
&& file_info.erasure.data_blocks > 0
&& file_info
.erasure
.data_blocks
.checked_add(file_info.erasure.parity_blocks)
.is_some_and(|shards| shards == disks.len()))
.then_some(file_info)
})
.collect::<Vec<_>>();
let Some(candidate) = candidates.first().copied() else {
// Without a data_dir + parts there is no data to prove recoverable.
if surviving.data_dir.is_none() || surviving.parts.is_empty() {
return Ok(false);
};
let identity = Self::file_info_quorum_hash(candidate);
if candidates
.iter()
.any(|file_info| Self::file_info_quorum_hash(file_info) != identity)
{
}
let data_blocks = surviving.erasure.data_blocks;
if data_blocks == 0 {
return Ok(false);
}
// Physically probe part presence on EVERY online disk using the surviving
// FileInfo's data_dir/parts. `check_parts` stats `object/<data_dir>/part.N`
// directly, so it counts disks that still hold the data even if their
// xl.meta was deleted.
let mut available = 0usize;
for disk in disks {
let Some(disk) = disk else {
return Ok(false);
};
match disk.check_parts(bucket, object, candidate).await {
Ok(response)
if !response.results.is_empty() && response.results.iter().all(|result| *result == CHECK_PART_SUCCESS) =>
{
available += 1;
}
Ok(_)
| Err(
DiskError::FileNotFound
| DiskError::FileVersionNotFound
| DiskError::PathNotFound
| DiskError::VolumeNotFound,
) => {}
Err(_) => return Ok(false),
for disk in disks.iter().flatten() {
if let Ok(resp) = disk.check_parts(bucket, object, &surviving).await
&& !resp.results.is_empty()
&& resp.results.iter().all(|r| *r == CHECK_PART_SUCCESS)
{
available += 1;
}
}
if available < candidate.erasure.data_blocks {
// Torn write: fewer than data_blocks surviving data shards is genuinely
// unrecoverable — preserve the current dangling behavior (no resurrection).
if available < data_blocks {
debug!(
bucket,
object,
available,
data_blocks,
"heal_object: version not reconstructable (torn write), keeping dangling behavior"
);
return Ok(false);
}
// Reconstructable: regenerate the surviving xl.meta on every disk whose
// metadata is absent so the version regains read-quorum. Each disk gets its
// OWN shard index: the disk at physical position `index` holds shard
// `distribution[index]` (mirrors `shuffle_disks` + the write path's
// `erasure.index = shuffled_pos + 1`). Copying the surviving disk's index
// verbatim would write an inconsistent xl.meta that the re-heal then treats
// as corrupt.
let distribution = &surviving.erasure.distribution;
let mut wrote = 0usize;
for (index, disk) in disks.iter().enumerate() {
let Some(disk) = disk else {
return Ok(false);
};
let metadata_absent = matches!(
let Some(disk) = disk else { continue };
let meta_absent = matches!(
errs.get(index).and_then(Option::as_ref),
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
);
if !metadata_absent {
) || !parts_metadata.get(index).map(FileInfo::is_valid).unwrap_or(false);
if !meta_absent {
continue;
}
let Some(&shard_index) = candidate.erasure.distribution.get(index) else {
return Ok(false);
// Without a known shard index for this position we cannot write a
// consistent xl.meta; leave it for the normal heal to reconstruct.
let Some(&shard_index) = distribution.get(index) else {
continue;
};
let mut regenerated = candidate.clone();
regenerated.fresh = false;
regenerated.erasure.index = shard_index;
match disk.write_metadata("", bucket, object, regenerated).await {
let mut regen = surviving.clone();
regen.fresh = false; // merge into any existing xl.meta on the disk
regen.erasure.index = shard_index;
match disk.write_metadata("", bucket, object, regen).await {
Ok(()) => wrote += 1,
Err(error) => {
Err(e) => {
warn!(
bucket,
object,
disk_index = index,
error = %error,
"failed to regenerate recoverable xl.meta"
error = %e,
"heal_object: failed to regenerate recoverable xl.meta on disk"
);
}
}
}
Ok(wrote > 0)
}
/// Best-effort orphan-data-dir reclaim for an object that is healthy on this
/// set. Wraps [`Self::reclaim_orphan_data_dirs`] with the shared logging so
/// both `heal_object` exits — the already-healthy early return and the
/// post-heal tail — reclaim identically. Never fails the heal: delete errors
/// are logged and swallowed. Callers must gate this on `!opts.dry_run`.
async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
match self.reclaim_orphan_data_dirs(bucket, object).await {
Ok(removed) if removed > 0 => {
info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
}
Ok(_) => {}
Err(e) => {
warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
}
}
}
/// Prevent dangling cleanup when surviving state cannot prove that deletion
/// is safe. Part presence proves only recoverability, never commit: the write
/// path can durably rename data before xl.meta is committed.
async fn dangling_delete_safety(
&self,
bucket: &str,
object: &str,
parts_metadata: &[FileInfo],
errs: &[Option<DiskError>],
disks: &[Option<DiskStore>],
) -> disk::error::Result<DanglingDeleteSafety> {
if disks.iter().any(Option::is_none)
|| errs.iter().flatten().any(|err| {
!matches!(
err,
DiskError::FileNotFound
| DiskError::FileVersionNotFound
| DiskError::PathNotFound
| DiskError::VolumeNotFound
)
})
{
return Ok(DanglingDeleteSafety::UnsafeToDelete);
if wrote == 0 {
return Ok(false);
}
let mut candidates = Vec::<RecoverableMetaCandidate>::with_capacity(parts_metadata.len());
for (fi, err) in parts_metadata.iter().zip(errs.iter()) {
if err.is_some() || !file_info_is_valid_for_metadata(fi) {
continue;
}
let identity = Self::file_info_quorum_hash(fi);
if !candidates.iter().any(|candidate| candidate.identity == identity) {
let local_payload = fi.has_valid_erasure_geometry()
&& !fi.deleted
&& !fi.is_remote()
&& fi.data_dir.is_some()
&& !fi.parts.is_empty()
&& fi.erasure.data_blocks > 0
&& fi
.erasure
.data_blocks
.checked_add(fi.erasure.parity_blocks)
.is_some_and(|shards| shards == disks.len());
candidates.push(RecoverableMetaCandidate {
identity,
file_info: fi.clone(),
data_count: 0,
local_payload,
});
}
}
if candidates
.iter()
.any(|candidate| candidate.file_info.deleted || candidate.file_info.is_remote())
|| candidates.len() > 1
{
return Ok(DanglingDeleteSafety::UnsafeToDelete);
}
for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
for (disk_index, disk) in disks.iter().enumerate() {
let Some(disk) = disk else {
return Ok(DanglingDeleteSafety::UnsafeToDelete);
};
#[cfg(test)]
let check_result = match injected_dangling_check_parts_error(bucket, object, disk_index) {
Some(error) => Err(error),
None => disk.check_parts(bucket, object, &candidate.file_info).await,
};
#[cfg(not(test))]
let check_result = disk.check_parts(bucket, object, &candidate.file_info).await;
match check_result {
Ok(resp) if !resp.results.is_empty() && resp.results.iter().all(|result| *result == CHECK_PART_SUCCESS) => {
candidate.data_count += 1;
}
Ok(_) => {}
Err(
DiskError::FileNotFound
| DiskError::FileVersionNotFound
| DiskError::PathNotFound
| DiskError::VolumeNotFound,
) => {}
Err(_) => return Ok(DanglingDeleteSafety::UnsafeToDelete),
}
}
}
Ok(
if candidates
.iter()
.any(|candidate| candidate.local_payload && candidate.data_count >= candidate.file_info.erasure.data_blocks)
{
DanglingDeleteSafety::UnsafeToDelete
} else {
DanglingDeleteSafety::NoRecoverableCandidate
},
)
info!(
bucket,
object,
available,
data_blocks,
regenerated_meta_disks = wrote,
"heal_object: rescued reconstructable sub-quorum version by regenerating xl.meta"
);
Ok(true)
}
pub(in crate::set_disk) async fn heal_object_dir_locked(
@@ -1537,7 +1393,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
#[cfg(test)]
mod heal_result_report_tests {
use super::{DanglingCheckPartsFailure, DanglingDeleteSafety, SetDisks};
use super::SetDisks;
use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
use crate::disk::endpoint::Endpoint;
use crate::disk::error::DiskError;
@@ -1549,12 +1405,10 @@ mod heal_result_report_tests {
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use crate::{config::storageclass, store::init_format::save_format_file};
use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo};
use std::sync::Arc;
use tempfile::TempDir;
use time::OffsetDateTime;
use tokio::sync::RwLock;
use uuid::Uuid;
async fn real_disk() -> (TempDir, Endpoint, DiskStore) {
let dir = tempfile::tempdir().expect("tempdir should be created");
@@ -1592,68 +1446,6 @@ mod heal_result_report_tests {
.await
}
fn meta_regen_test_fileinfo(object: &str, data_dir: Uuid, mod_time: i64, disk_index: usize) -> FileInfo {
let mut fi = FileInfo::new(object, 2, 2);
fi.data_dir = Some(data_dir);
fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(mod_time).expect("test timestamp should parse"));
fi.size = 1;
fi.parts = vec![ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
}];
fi.erasure.index = fi.erasure.distribution[disk_index];
fi
}
async fn meta_regen_test_set(
bucket: &str,
object: &str,
data_dirs: &[(Uuid, usize)],
) -> (Vec<TempDir>, Arc<SetDisks>, Vec<Option<DiskStore>>) {
let mut temp_dirs = Vec::new();
let mut endpoints = Vec::new();
let mut disks = Vec::new();
for disk_index in 0..4 {
let (temp_dir, endpoint, disk) = real_disk().await;
disk.make_volume(bucket).await.expect("test bucket should be created");
for (data_dir, shard_count) in data_dirs {
if disk_index >= *shard_count {
continue;
}
let part_dir = temp_dir.path().join(bucket).join(object).join(data_dir.to_string());
tokio::fs::create_dir_all(&part_dir)
.await
.expect("test data directory should be created");
tokio::fs::write(part_dir.join("part.1"), [1u8; 2])
.await
.expect("test data shard should be written");
}
temp_dirs.push(temp_dir);
endpoints.push(endpoint);
disks.push(Some(disk));
}
let set = set_disks_with(disks.clone(), endpoints, 2).await;
(temp_dirs, set, disks)
}
async fn seed_meta_regen_test_metadata(
disks: &[Option<DiskStore>],
disk_index: usize,
bucket: &str,
object: &str,
file_info: &FileInfo,
) {
disks[disk_index]
.as_ref()
.expect("metadata test disk should be online")
.write_metadata("", bucket, object, file_info.clone())
.await
.expect("test metadata should be written");
}
async fn formatted_single_disk_no_parity_set() -> (TempDir, Arc<SetDisks>) {
let format = FormatV3::new(1, 1);
let dir = tempfile::tempdir().expect("tempdir should be created");
@@ -1961,312 +1753,6 @@ mod heal_result_report_tests {
assert_eq!(result.before.drives[3].state, DriveState::Ok.to_string());
}
#[tokio::test]
async fn dangling_delete_guard_preserves_conflicting_identities_without_writing_metadata() {
let bucket = "bucket-delete-guard-conflict";
let object = "object.bin";
let old_data_dir = Uuid::parse_str("99999999-9999-9999-9999-999999999999").expect("old data dir should parse");
let new_data_dir = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa").expect("new data dir should parse");
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(old_data_dir, 4), (new_data_dir, 2)]).await;
let version_id = Uuid::parse_str("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb").expect("version id should parse");
let mut metadata = vec![
meta_regen_test_fileinfo(object, old_data_dir, 9, 0),
meta_regen_test_fileinfo(object, new_data_dir, 10, 1),
FileInfo::default(),
FileInfo::default(),
];
metadata[0].version_id = Some(version_id);
metadata[1].version_id = Some(version_id);
assert_eq!(
metadata[0].version_id, metadata[1].version_id,
"the conflicting candidates must share one version id"
);
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata[0]).await;
seed_meta_regen_test_metadata(&disks, 1, bucket, object, &metadata[1]).await;
let errs = vec![None, None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)];
assert!(
set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
.await
.expect("conflicting identities should be classified")
== DanglingDeleteSafety::UnsafeToDelete
);
let reversed = vec![
metadata[1].clone(),
metadata[0].clone(),
FileInfo::default(),
FileInfo::default(),
];
assert!(
set.dangling_delete_safety(bucket, object, &reversed, &errs, &disks)
.await
.expect("reversed identities should be classified")
== DanglingDeleteSafety::UnsafeToDelete
);
let version_id = version_id.to_string();
assert!(
!set.try_regenerate_explicit_version_meta(bucket, object, &version_id, &metadata, &errs, &disks)
.await
.expect("conflicting explicit-version candidates should be rejected"),
"an explicit version must not select between conflicting metadata identities"
);
for disk_index in [2, 3] {
assert!(
matches!(
disks[disk_index]
.as_ref()
.expect("test disk should be online")
.read_version("", bucket, object, "", &ReadOptions::default())
.await,
Err(DiskError::FileNotFound)
),
"the delete guard must not manufacture metadata on missing disks"
);
}
let old = disks[0]
.as_ref()
.expect("first test disk should be online")
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("old metadata should remain readable");
let new = disks[1]
.as_ref()
.expect("second test disk should be online")
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("new metadata should remain readable");
assert_eq!(old.data_dir, Some(old_data_dir));
assert_eq!(new.data_dir, Some(new_data_dir));
}
#[tokio::test]
async fn heal_meta_quorum_failure_preserves_reconstructable_uncommitted_candidate() {
let bucket = "bucket-delete-guard-reconstructable";
let object = "object.bin";
let data_dir = Uuid::parse_str("33333333-3333-3333-3333-333333333333").expect("data dir should parse");
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
let metadata = [
meta_regen_test_fileinfo(object, data_dir, 3, 0),
FileInfo::default(),
FileInfo::default(),
FileInfo::default(),
];
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata[0]).await;
let (observed_metadata, observed_errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", true, true, false)
.await
.expect("test metadata should be readable across the set");
assert_eq!(
set.dangling_delete_safety(bucket, object, &observed_metadata, &observed_errs, &disks)
.await
.expect("observed reconstructable candidate should be classified"),
DanglingDeleteSafety::UnsafeToDelete
);
let (_, err) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
no_lock: true,
..Default::default()
},
)
.await
.expect("unsafe dangling state should be reported without deletion");
assert_eq!(err, Some(DiskError::FileNotFound));
let surviving = disks[0]
.as_ref()
.expect("first test disk should be online")
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("the only metadata copy must be preserved");
assert_eq!(surviving.data_dir, Some(data_dir));
assert!(
matches!(
disks[1]
.as_ref()
.expect("second test disk should be online")
.read_version("", bucket, object, "", &ReadOptions::default())
.await,
Err(DiskError::FileNotFound)
),
"the delete guard must not propagate metadata"
);
}
#[tokio::test]
async fn heal_meta_quorum_failure_preserves_candidate_when_required_shard_disk_is_offline() {
let bucket = "bucket-delete-guard-offline";
let object = "object.bin";
let data_dir = Uuid::parse_str("44444444-4444-4444-4444-444444444444").expect("data dir should parse");
let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
let metadata = meta_regen_test_fileinfo(object, data_dir, 4, 0);
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
set.disks.write().await[1] = None;
let (_, err) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
no_lock: true,
..Default::default()
},
)
.await
.expect("offline shard state should be reported without deletion");
assert_eq!(err, Some(DiskError::FileNotFound));
let surviving = disks[0]
.as_ref()
.expect("first test disk should be online")
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("offline uncertainty must preserve the surviving metadata");
assert_eq!(surviving.data_dir, Some(data_dir));
assert!(
temp_dirs[0]
.path()
.join(bucket)
.join(object)
.join(data_dir.to_string())
.join("part.1")
.is_file(),
"offline uncertainty must preserve the last online shard"
);
}
#[tokio::test]
async fn heal_meta_quorum_failure_preserves_candidate_when_part_probe_times_out() {
let bucket = "bucket-delete-guard-timeout";
let object = "object.bin";
let data_dir = Uuid::parse_str("55555555-5555-5555-5555-555555555555").expect("data dir should parse");
let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
let metadata = meta_regen_test_fileinfo(object, data_dir, 5, 0);
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
let _failure = DanglingCheckPartsFailure::install(bucket, object, 1, DiskError::Timeout);
let (_, err) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
no_lock: true,
..Default::default()
},
)
.await
.expect("part probe timeout should be reported without deletion");
assert_eq!(err, Some(DiskError::FileNotFound));
let surviving = disks[0]
.as_ref()
.expect("first test disk should be online")
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("probe uncertainty must preserve the surviving metadata");
assert_eq!(surviving.data_dir, Some(data_dir));
assert!(
temp_dirs[0]
.path()
.join(bucket)
.join(object)
.join(data_dir.to_string())
.join("part.1")
.is_file(),
"probe uncertainty must preserve the last confirmed shard"
);
}
#[tokio::test]
async fn dangling_delete_guard_ignores_set_incompatible_geometry() {
let bucket = "bucket-delete-guard-short-geometry";
let object = "object.bin";
let data_dir = Uuid::parse_str("abababab-abab-abab-abab-abababababab").expect("data dir should parse");
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 1)]).await;
let mut candidate = FileInfo::new(object, 1, 0);
candidate.data_dir = Some(data_dir);
candidate.mod_time = Some(OffsetDateTime::from_unix_timestamp(18).expect("timestamp should parse"));
candidate.size = 1;
candidate.parts = vec![ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
}];
candidate.erasure.index = candidate.erasure.distribution[0];
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &candidate).await;
let metadata = vec![candidate, FileInfo::default(), FileInfo::default(), FileInfo::default()];
let errs = vec![
None,
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
];
assert!(
set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
.await
.expect("set-incompatible geometry should be classified")
== DanglingDeleteSafety::NoRecoverableCandidate
);
}
#[tokio::test]
async fn dangling_delete_guard_preserves_delete_marker_and_remote_metadata() {
let bucket = "bucket-delete-guard-nonlocal";
let object = "object.bin";
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
let marker = FileInfo {
name: object.to_string(),
version_id: Some(Uuid::parse_str("eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee").expect("version id should parse")),
deleted: true,
mod_time: Some(OffsetDateTime::from_unix_timestamp(14).expect("marker timestamp should parse")),
..Default::default()
};
let remote_dir = Uuid::parse_str("89898989-8989-8989-8989-898989898989").expect("remote data dir should parse");
let mut remote = meta_regen_test_fileinfo(object, remote_dir, 15, 1);
remote.transition_status = TRANSITION_COMPLETE.to_string();
remote.transition_tier = "WARM".to_string();
remote.transitioned_objname = "remote/object.bin".to_string();
for metadata in [marker, remote] {
let candidates = vec![metadata, FileInfo::default(), FileInfo::default(), FileInfo::default()];
let errs = vec![
None,
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
];
assert_eq!(
set.dangling_delete_safety(bucket, object, &candidates, &errs, &disks)
.await
.expect("non-local metadata should be classified"),
DanglingDeleteSafety::UnsafeToDelete
);
}
}
#[tokio::test]
async fn dangling_delete_guard_preserves_metadata_read_uncertainty() {
let bucket = "bucket-delete-guard-read-error";
let object = "object.bin";
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
let metadata = vec![FileInfo::default(); disks.len()];
for read_error in [DiskError::Timeout, DiskError::DiskAccessDenied, DiskError::DiskNotFound] {
let mut errs = vec![Some(DiskError::FileNotFound); disks.len()];
errs[0] = Some(read_error);
assert_eq!(
set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
.await
.expect("metadata read uncertainty should be classified"),
DanglingDeleteSafety::UnsafeToDelete
);
}
}
#[tokio::test]
async fn heal_no_parity_bitrot_reports_unrecoverable_integrity_failure() {
let (dir, set) = formatted_single_disk_no_parity_set().await;
+88 -134
View File
@@ -27,7 +27,7 @@ use crate::multipart_listing::paginate_multipart_listing;
use futures::{StreamExt, stream};
use std::future::Future;
#[cfg(test)]
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::task::JoinSet;
@@ -36,7 +36,6 @@ const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
#[cfg(test)]
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum MultipartCommitPause {
PutPartBeforeLockAcquire,
PutPartBeforeLockLost,
PutPartAfterRename,
BeforeLockLost,
@@ -48,10 +47,9 @@ struct MultipartCommitBarrierState {
bucket: String,
object: String,
pause: MultipartCommitPause,
expected_arrivals: usize,
arrivals: AtomicUsize,
armed: AtomicBool,
arrived: tokio::sync::Notify,
release: tokio::sync::Semaphore,
release: tokio::sync::Notify,
}
#[cfg(test)]
@@ -66,24 +64,13 @@ static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc
#[cfg(test)]
impl MultipartCommitBarrier {
pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
Self::install_for_arrivals(bucket, object, pause, 1)
}
pub(crate) fn install_for_arrivals(
bucket: &str,
object: &str,
pause: MultipartCommitPause,
expected_arrivals: usize,
) -> Self {
assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival");
let state = Arc::new(MultipartCommitBarrierState {
bucket: bucket.to_string(),
object: object.to_string(),
pause,
expected_arrivals,
arrivals: AtomicUsize::new(0),
armed: AtomicBool::new(true),
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Semaphore::new(0),
release: tokio::sync::Notify::new(),
});
let mut slot = MULTIPART_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -96,28 +83,20 @@ impl MultipartCommitBarrier {
}
pub(crate) async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), async {
loop {
let arrived = self.state.arrived.notified();
if self.state.arrivals.load(Ordering::Acquire) >= self.state.expected_arrivals {
return;
}
arrived.await;
}
})
.await
.expect("multipart completion should reach the deterministic commit barrier");
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("multipart completion should reach the deterministic commit barrier");
}
pub(crate) fn release(&self) {
self.state.release.add_permits(self.state.expected_arrivals);
self.state.release.notify_one();
}
}
#[cfg(test)]
impl Drop for MultipartCommitBarrier {
fn drop(&mut self) {
self.release();
self.state.release.notify_one();
let mut slot = MULTIPART_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
@@ -138,20 +117,10 @@ async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartComm
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
.cloned();
if let Some(barrier) = barrier
&& let Ok(previous) = barrier.arrivals.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
(current < barrier.expected_arrivals).then_some(current + 1)
})
&& barrier.armed.swap(false, Ordering::AcqRel)
{
let arrival = previous + 1;
if arrival == barrier.expected_arrivals {
barrier.arrived.notify_one();
}
barrier
.release
.acquire()
.await
.expect("multipart commit barrier should remain open")
.forget();
barrier.arrived.notify_one();
barrier.release.notified().await;
}
}
@@ -724,8 +693,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
#[cfg(test)]
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockAcquire).await;
// Serialize only the commit (rename_part), not the whole upload. Each
// concurrent stream writes to its own unique temp dir (see `tmp_part`
// above), so the encode/stream phase never conflicts and must stay
@@ -3379,98 +3346,85 @@ mod tests {
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_holds_object_then_upload_lock_through_commit() {
temp_env::async_with_vars(
[
(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
],
async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "multipart-layout-lock-order-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let create_opts = ObjectOptions {
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
..Default::default()
};
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
signaling.set_target(upload_resource.clone());
signaling.clear_observed();
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "multipart-layout-lock-order-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let create_opts = ObjectOptions {
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
..Default::default()
};
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
signaling.set_target(upload_resource.clone());
signaling.clear_observed();
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
let observed = signaling.observed();
let object_position = observed
.iter()
.position(|resource| resource == &object_resource)
.expect("completion should acquire the object lock");
let upload_position = observed
.iter()
.position(|resource| resource == &upload_resource)
.expect("completion should acquire the upload lock");
assert!(object_position < upload_position, "completion must acquire object before upload");
let abort_store = set_disks.clone();
let abort_upload_id = upload_id.clone();
let abort = tokio::spawn(async move {
abort_store
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(2).await;
tokio::task::yield_now().await;
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
let list_store = set_disks.clone();
let list_upload_id = upload_id.clone();
let list = tokio::spawn(async move {
list_store
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(3).await;
tokio::task::yield_now().await;
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
barrier.release();
complete
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
.expect("completion task should not panic")
.expect("completion should commit after the barrier is released");
let abort_err = abort
});
barrier.wait_until_paused().await;
let observed = signaling.observed();
let object_position = observed
.iter()
.position(|resource| resource == &object_resource)
.expect("completion should acquire the object lock");
let upload_position = observed
.iter()
.position(|resource| resource == &upload_resource)
.expect("completion should acquire the upload lock");
assert!(object_position < upload_position, "completion must acquire object before upload");
let abort_store = set_disks.clone();
let abort_upload_id = upload_id.clone();
let abort = tokio::spawn(async move {
abort_store
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
.await
.expect("abort task should not panic")
.expect_err("the committed upload should no longer exist when abort acquires the lock");
assert!(
matches!(abort_err, StorageError::InvalidUploadID(..)),
"abort should return InvalidUploadID after completion, got {abort_err:?}"
);
let list_err = list
});
signaling.wait_for_attempts(2).await;
tokio::task::yield_now().await;
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
let list_store = set_disks.clone();
let list_upload_id = upload_id.clone();
let list = tokio::spawn(async move {
list_store
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.await
.expect("ListParts task should not panic")
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
assert!(
matches!(list_err, StorageError::InvalidUploadID(..)),
"ListParts should return InvalidUploadID after completion, got {list_err:?}"
);
},
)
});
signaling.wait_for_attempts(3).await;
tokio::task::yield_now().await;
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
barrier.release();
complete
.await
.expect("completion task should not panic")
.expect("completion should commit after the barrier is released");
let abort_err = abort
.await
.expect("abort task should not panic")
.expect_err("the committed upload should no longer exist when abort acquires the lock");
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
let list_err = list
.await
.expect("ListParts task should not panic")
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
})
.await;
}
+54 -264
View File
@@ -42,7 +42,6 @@ use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppress
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
use crate::store::ECStore;
use futures::FutureExt as _;
use http::HeaderValue;
use std::future::Future;
fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
@@ -50,17 +49,6 @@ fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Er
.map_err(Error::from)
}
async fn get_object_reader_with_context(
ctx: &InstanceContext,
reader: Box<dyn AsyncRead + Unpin + Send + Sync>,
range: Option<HTTPRangeSpec>,
object_info: &ObjectInfo,
opts: &ObjectOptions,
headers: &HeaderMap<HeaderValue>,
) -> Result<(GetObjectReader, usize, i64)> {
GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await
}
/// Length of the full plaintext body when — and only when — this read's output
/// is exactly the object's complete plaintext, so the app-layer body cache may
/// serve it in place of the erasure read.
@@ -562,7 +550,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
&self.ctx.tier_config_mgr(),
)
.await?;
return Ok(finish_set_disk_read_lock(gr, read_lock_guard.take(), None, bucket, object));
return Ok(finish_set_disk_read_lock(
gr,
read_lock_guard.take(),
lock_optimization_enabled,
bucket,
object,
));
}
// App-layer object data cache probe: metadata (etag/size) is resolved
@@ -689,18 +683,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
return Ok(reader);
}
let snapshot_lease = if lock_optimization_enabled {
match fi.data_dir.filter(|data_dir| !data_dir.is_nil()) {
Some(data_dir) => {
let data_dir_path = format!("{object}/{data_dir}");
acquire_snapshot_leases(&disks, bucket, &data_dir_path, fi.erasure.data_blocks).await
}
None => None,
}
} else {
None
};
match codec_streaming_gate.decision {
GetCodecStreamingDecision::Use => {
match Self::get_object_decode_reader_with_fileinfo(
@@ -725,12 +707,17 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
size_bucket,
);
record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket);
let (mut reader, _offset, _length) =
get_object_reader_with_context(&self.ctx, stream, range, &object_info, opts, &h).await?;
let (mut reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?;
// Carry the hook probe result so the app layer skips its
// now-redundant lookup on the streaming miss path (ODC-16).
reader.body_source = body_source;
return Ok(finish_set_disk_read_lock(reader, read_lock_guard.take(), snapshot_lease, bucket, object));
return Ok(finish_set_disk_read_lock(
reader,
read_lock_guard.take(),
lock_optimization_enabled,
bucket,
object,
));
}
core::io_primitives::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
record_get_codec_streaming_gate_decision(
@@ -758,8 +745,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
let (rd, wd) = tokio::io::duplex(duplex_buffer_size);
debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer");
let (mut reader, offset, length) =
get_object_reader_with_context(&self.ctx, Box::new(rd), range, &object_info, opts, &h).await?;
let (mut reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h).await?;
// Carry the hook probe result so the app layer skips its now-redundant
// lookup on the streaming miss path (ODC-16).
reader.body_source = body_source;
@@ -770,22 +756,15 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
let set_index = self.set_index;
let pool_index = self.pool_index;
let skip_verify = opts.skip_verify_bitrot;
let producer_snapshot_lease = snapshot_lease.clone();
if let Some(lease) = snapshot_lease {
if lock_optimization_enabled {
release_materialized_read_lock(&bucket, &object, read_lock_guard.take());
reader.stream = Box::new(SnapshotLeaseReader {
inner: reader.stream,
lease: Some(lease),
terminal_error: false,
});
debug!(bucket, object, "Lock optimization: replaced read lock with snapshot leases");
debug!(bucket, object, "Lock optimization: released read lock before streaming read");
}
// The producer shares the lease lifetime with the body so cancellation
// cannot release the snapshot while the duplex task is still unwinding.
// When lock optimization is disabled, keep the read-lock guard in the
// task so it lives for the duration of the streaming read.
tokio::spawn(async move {
let _guard = read_lock_guard;
let _snapshot_lease = producer_snapshot_lease;
let mut writer = GetObjectDownstreamWriter::new(wd);
// Do not wrap the entire read+write pipeline in `disk_read_timeout`.
// `get_object_with_fileinfo` also waits on `writer`, so an outer timeout
@@ -2334,78 +2313,25 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
}
}
#[cfg(test)]
fn persisted_transition_version(
remote_version: &str,
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
}
#[cfg(test)]
fn remote_version_state_writer_enabled() -> bool {
remote_version_state_writer_fleet_proof().is_some()
}
fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
remote_version_state_writer_requested()
.then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
.flatten()
}
fn remote_version_state_writer_requested() -> bool {
remote_version_state_writer_enabled_for(
rustfs_utils::get_env_bool(
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE,
),
rustfs_utils::get_env_bool(
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
),
true,
)
}
fn remote_version_state_writer_fleet_proof_matches(
proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
) -> bool {
remote_version_state_writer_fleet_proof_matches_for(
remote_version_state_writer_requested(),
crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
)
}
fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
requested && fleet_proof_matches
}
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
requested && fleet_confirmed && fleet_proof_valid
}
fn persisted_transition_version_with_gate(
remote_version: &str,
remote_version_state_writer_enabled: bool,
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
if remote_version.is_empty() {
return Ok((None, rustfs_filemeta::TransitionVersionState::KnownDisabled));
}
match Uuid::parse_str(remote_version) {
Ok(version_id) if version_id.is_nil() => Err(std::io::Error::new(
let version_id = Uuid::parse_str(remote_version).map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::Unsupported,
"opaque remote tier versions require the cluster capability gate",
)
})?;
if version_id.is_nil() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote tier returned a nil object version ID",
)),
Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"opaque remote tier versions require the operator-attested live fleet capability gate",
)),
Err(_) if remote_version == "null" => {
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
}
Err(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
));
}
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact))
}
#[cfg(test)]
@@ -2643,10 +2569,7 @@ mod transition_upload_completion_tests {
#[cfg(test)]
mod transition_version_id_tests {
use super::{
TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
};
use super::{TransitionUploadCandidate, persisted_transition_version};
use rustfs_filemeta::TransitionVersionState;
use uuid::Uuid;
@@ -2685,64 +2608,6 @@ mod transition_version_id_tests {
"opaque-version-token"
);
}
#[test]
fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
("old defaults", false, false, false, false),
("missing fleet confirmation", true, false, true, false),
("missing local opt-in", false, true, true, false),
("missing fleet proof", true, true, false, false),
("explicitly unconfirmed fleet", true, false, true, false),
("rolled-back writer", false, true, true, false),
("fully upgraded fleet", true, true, true, true),
] {
assert_eq!(
remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
expected,
"{case}"
);
}
}
#[test]
fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
for (case, requested, fleet_proof_matches, expected) in [
("operator gate closed", false, true, false),
("fleet proof changed", true, false, false),
("current authorization", true, true, true),
] {
assert_eq!(
remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
expected,
"{case}"
);
}
}
#[test]
fn fleet_gate_enables_null_and_opaque_remote_version_states() {
for (remote_version, expected) in [
("null", (Some("null".to_string()), TransitionVersionState::SuspendedNull)),
(
"opaque-version-token",
(Some("opaque-version-token".to_string()), TransitionVersionState::Exact),
),
] {
assert!(
persisted_transition_version_with_gate(remote_version, false).is_err(),
"missing fleet confirmation must reject {remote_version:?}"
);
assert_eq!(
persisted_transition_version_with_gate(remote_version, true).expect("fleet-confirmed state must be persisted"),
expected
);
}
assert_eq!(
persisted_transition_version_with_gate("", true).expect("empty remote version identifies an unversioned tier"),
(None, TransitionVersionState::KnownDisabled)
);
}
}
impl SetDisks {
@@ -3901,16 +3766,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let dest_obj = transaction.remote_object.clone();
let mut transition_meta = (*oi.user_defined).clone();
transition_meta.insert("name".to_string(), object.to_string());
rustfs_utils::http::metadata_compat::insert_str(
&mut transition_meta,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
transaction.transaction_id.to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut transition_meta,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
rustfs_utils::crypto::hex(transaction.backend_fingerprint),
);
if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
@@ -4021,24 +3876,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
return Err(err.into());
}
let fleet_proof = remote_version_state_writer_fleet_proof();
let remote_version_requires_fleet_proof =
!candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
let (transition_version_id, transition_version_state) =
match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
Ok(version) => version,
Err(err) => {
let cleanup_api = transition_cleanup_store(&self.ctx).await;
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
return Err(StorageError::Io(std::io::Error::other(format!(
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
))));
}
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
.await;
return Err(err.into());
let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) {
Ok(version) => version,
Err(err) => {
let cleanup_api = transition_cleanup_store(&self.ctx).await;
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
return Err(StorageError::Io(std::io::Error::other(format!(
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
))));
}
};
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
.await;
return Err(err.into());
}
};
if let Err(err) = advance_and_save_transition_transaction(
transaction_api.as_ref(),
&mut transaction,
@@ -4154,21 +4005,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
#[cfg(test)]
pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
// This check is the fleet-proof lease linearization point. Revocation
// blocks later commits; an already-authorized local quorum commit is
// allowed to finish without holding a synchronous lock across I/O.
if remote_version_requires_fleet_proof
&& !fleet_proof
.as_ref()
.is_some_and(remote_version_state_writer_fleet_proof_matches)
{
drop(transition_lock_guard);
if upload_cleanup.cleanup().await.is_ok() {
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
.await;
}
return Err(Error::other("remote version state fleet capability changed during transition"));
}
if let Err(err) = advance_and_save_transition_transaction(
transaction_api.as_ref(),
&mut transaction,
@@ -4550,61 +4386,6 @@ mod erasure_construction_tests {
}
}
#[cfg(test)]
mod object_encryption_resolver_wiring_tests {
use super::*;
use crate::object_api::{EncryptionResolutionError, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionRequest};
use std::io::Cursor;
use std::sync::atomic::{AtomicUsize, Ordering};
struct CountingResolver {
calls: AtomicUsize,
}
#[async_trait::async_trait]
impl ObjectEncryptionResolver for CountingResolver {
async fn resolve_read_material(
&self,
_request: ReadEncryptionRequest<'_>,
) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
self.calls.fetch_add(1, Ordering::Relaxed);
Ok(None)
}
}
#[tokio::test]
async fn get_object_reader_forwards_instance_resolver() {
let resolver = Arc::new(CountingResolver {
calls: AtomicUsize::new(0),
});
let ctx = InstanceContext::new();
assert!(
ctx.set_object_encryption_resolver(resolver.clone()).is_ok(),
"fresh context should accept resolver"
);
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 1,
user_defined: Arc::new(HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])),
..Default::default()
};
let result = get_object_reader_with_context(
&ctx,
Box::new(Cursor::new(Vec::<u8>::new())),
None,
&object_info,
&ObjectOptions::default(),
&HeaderMap::new(),
)
.await;
assert!(result.is_err(), "resolver returning no material must fail closed");
assert_eq!(resolver.calls.load(Ordering::Relaxed), 1);
}
}
#[cfg(test)]
pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
//! Shared hermetic `SetDisks` construction for the ops tests below: the
@@ -7385,6 +7166,7 @@ mod transition_source_identity_matrix_tests {
async fn transition_source_identity_field_matrix_rejects_single_field_drift() {
#[derive(Clone, Copy, Debug)]
enum IdentityField {
VersionId,
DataDir,
ModTime,
Size,
@@ -7400,6 +7182,7 @@ mod transition_source_identity_matrix_tests {
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
for (index, field) in [
IdentityField::VersionId,
IdentityField::DataDir,
IdentityField::ModTime,
IdentityField::Size,
@@ -7453,6 +7236,10 @@ mod transition_source_identity_matrix_tests {
let mut changed = source.clone();
match field {
IdentityField::VersionId => {
changed.version_id = Some(Uuid::new_v4());
changed.fresh = true;
}
IdentityField::DataDir => changed.data_dir = Some(Uuid::new_v4()),
IdentityField::ModTime => {
changed.mod_time = changed.mod_time.map(|value| value + time::Duration::nanoseconds(1));
@@ -7493,12 +7280,15 @@ mod transition_source_identity_matrix_tests {
);
match field {
IdentityField::VersionId => assert_ne!(source.version_id, persisted.version_id),
IdentityField::DataDir => assert_ne!(source.data_dir, persisted.data_dir),
IdentityField::ModTime => assert_ne!(source.mod_time, persisted.mod_time),
IdentityField::Size => assert_ne!(source.size, persisted.size),
IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&persisted.metadata)),
}
assert_eq!(source.version_id, persisted.version_id);
if !matches!(field, IdentityField::VersionId) {
assert_eq!(source.version_id, persisted.version_id);
}
if !matches!(field, IdentityField::DataDir) {
assert_eq!(source.data_dir, persisted.data_dir);
}
+2 -53
View File
@@ -87,7 +87,7 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
}
pub(crate) async fn await_bucket_namespace_operation<T, F>(
async fn await_bucket_namespace_operation<T, F>(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
operation: &'static str,
@@ -568,7 +568,6 @@ mod tests {
};
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
use crate::bucket::metadata_sys;
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::error::StorageError;
use crate::object_api::{ObjectOptions, PutObjReader};
@@ -590,7 +589,6 @@ mod tests {
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, SystemTime};
use time::OffsetDateTime;
use tokio::io::AsyncReadExt;
use tokio::sync::{Notify, OnceCell};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
@@ -1050,7 +1048,7 @@ mod tests {
.await
.expect("bucket should be created");
for disk_path in &disk_paths {
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory"))
.await
.expect("empty directory remnant should be created");
}
@@ -1259,55 +1257,6 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn bucket_delete_preserves_put_committed_after_empty_scan() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
let object = "committed-after-empty-scan";
let payload = b"object committed after DeleteBucket empty scan".to_vec();
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let barrier = install_delete_bucket_empty_scan_barrier();
let delete_store = ecstore.clone();
let delete_bucket = bucket.clone();
let delete = tokio::spawn(async move {
delete_store
.delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
.await
});
barrier.wait_until_paused().await;
let mut put_reader = PutObjReader::from_vec(payload.clone());
ecstore
.put_object(&bucket, object, &mut put_reader, &ObjectOptions::default())
.await
.expect("PUT should commit while DeleteBucket is paused after its empty scan");
barrier.release();
let err = delete
.await
.expect("DeleteBucket task should join")
.expect_err("DeleteBucket must reject a PUT committed after its empty scan");
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
let mut reader = ecstore
.get_object_reader(&bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
.await
.expect("committed object should remain readable after DeleteBucket fails");
let mut restored = Vec::new();
reader
.stream
.read_to_end(&mut restored)
.await
.expect("object body should remain readable");
assert_eq!(restored, payload);
}
#[tokio::test]
#[serial]
async fn bucket_recreation_does_not_publish_unverified_usage() {
+6 -191
View File
@@ -562,12 +562,10 @@ mod tests {
},
tier_sweeper::Jentry,
transition_transaction::{
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
inspect_transition_transaction_for_operator, load_transition_transaction_record,
recover_transition_transaction_records, save_transition_transaction_record,
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
save_transition_transaction_record,
},
},
client::transition_api::ReaderImpl,
@@ -577,7 +575,6 @@ mod tests {
services::tier::{
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
tier::{TIER_CONFIG_FILE, TierConfigMgr},
tier_config::{TierConfig, TierType, TierWasabi},
tier_mutation_intent::{
TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
@@ -1166,37 +1163,6 @@ mod tests {
.len()
}
#[cfg(feature = "test-util")]
async fn register_transition_reconcile_test_tier(
handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
tier_name: &str,
) -> MockWarmBackend {
let backend = MockWarmBackend::new();
let mut manager = handle.write().await;
manager.tiers.insert(
tier_name.to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::Wasabi,
name: tier_name.to_string(),
wasabi: Some(TierWasabi {
name: tier_name.to_string(),
endpoint: "https://s3.wasabisys.com".to_string(),
access_key: "test-access-key".to_string(),
secret_key: "test-secret-key".to_string(),
bucket: "mock-tier".to_string(),
prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
region: "us-east-1".to_string(),
}),
..Default::default()
},
);
manager
.install_test_driver(tier_name, Box::new(backend.clone()))
.expect("mock fallback tier driver should install");
backend
}
#[cfg(feature = "test-util")]
async fn wait_for_tier_delete_journal_recovery(
store: Arc<crate::store::ECStore>,
@@ -2772,7 +2738,7 @@ mod tests {
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
@@ -2843,157 +2809,6 @@ mod tests {
}
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXOPERATOR";
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
.backend_identity();
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
transaction_id: uuid::Uuid::new_v4(),
owner_epoch: uuid::Uuid::new_v4(),
write_id: uuid::Uuid::new_v4(),
source: TransitionSourceIdentity {
bucket: "source-bucket".to_string(),
object: "source-object".to_string(),
version_id: None,
data_dir: uuid::Uuid::new_v4(),
mod_time_unix_nanos: 1_770_000_000_000_000_000,
size: 42,
etag: "source-etag".to_string(),
version_mode: TransitionSourceVersionMode::Unversioned,
},
tier_name: tier_name.to_string(),
backend_fingerprint: backend_identity,
not_after_unix_nanos: 1,
})
.expect("transaction should build");
transaction
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
.expect("transaction should enter unknown upload outcome state");
let remote_version = uuid::Uuid::new_v4().to_string();
backend.set_put_remote_version(Some(remote_version.clone())).await;
let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
backend
.put(
&transaction.remote_object,
ReaderImpl::Body(candidate.clone()),
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
)
.await
.expect("mock backend should accept candidate");
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
.await
.expect("operator inspection should probe the candidate");
assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
let wrong_version = uuid::Uuid::new_v4().to_string();
let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
.await
.expect_err("a mismatched exact version must fail before deleting a candidate");
assert!(matches!(
err,
TransitionOperatorError::CandidateVersionMismatch {
expected,
actual: TransitionOperatorProbe::VersionedPresent(ref observed),
} if expected == wrong_version && observed == &remote_version
));
assert!(backend.contains(&transaction.remote_object).await);
assert_eq!(backend.exact_remove_count(), 0);
load_transition_transaction_record(store.clone(), transaction.transaction_id)
.await
.expect("an incorrect exact version must retain the transaction journal");
let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
.await
.expect("operator-confirmed exact candidate should be deleted");
assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
assert!(result.journal_observed_after_delete);
assert_eq!(backend.exact_remove_count(), 1);
assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
load_transition_transaction_record(store.clone(), transaction.transaction_id)
.await
.expect("candidate deletion must retain the transaction journal");
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
.await
.expect("a separately confirmed missing candidate should permit finalization");
assert!(matches!(
load_transition_transaction_record(store, transaction.transaction_id).await,
Err(Error::ConfigNotFound)
));
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn operator_finalize_retains_record_without_missing_proof() {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXOPERATORFAIL";
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
.backend_identity();
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
transaction_id: uuid::Uuid::new_v4(),
owner_epoch: uuid::Uuid::new_v4(),
write_id: uuid::Uuid::new_v4(),
source: TransitionSourceIdentity {
bucket: "source-bucket".to_string(),
object: "source-object".to_string(),
version_id: None,
data_dir: uuid::Uuid::new_v4(),
mod_time_unix_nanos: 1_770_000_000_000_000_000,
size: 42,
etag: "source-etag".to_string(),
version_mode: TransitionSourceVersionMode::Unversioned,
},
tier_name: tier_name.to_string(),
backend_fingerprint: backend_identity,
not_after_unix_nanos: 1,
})
.expect("transaction should build");
transaction
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
.expect("transaction should enter unknown upload outcome state");
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
backend
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
.await;
assert!(matches!(
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
));
load_transition_transaction_record(store, transaction.transaction_id)
.await
.expect("an unsupported probe must retain the transaction journal");
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
@@ -3004,7 +2819,7 @@ mod tests {
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXRESPONSELOSS";
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let bucket = "transition-response-loss-bucket";
let object = "source.bin";
store
-1
View File
@@ -6895,7 +6895,6 @@ mod test {
)
.await
.expect("local disk should be created");
disk.make_volume("bucket").await.expect("test bucket should be created");
let mut fi = FileInfo::new("object", 1, 1);
fi.volume = "bucket".to_owned();
fi.name = "object".to_owned();
-1
View File
@@ -141,7 +141,6 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
const MAX_UPLOADS_LIST: usize = 10000;
mod bucket;
pub(crate) use bucket::await_bucket_namespace_operation;
mod heal;
mod heal_walk;
pub use heal_walk::HealWalkVersion;
-43
View File
@@ -1,43 +0,0 @@
//! Test-only guard against tracing's process-global callsite-interest cache.
//!
//! Any test that installs its own subscriber and then asserts on span or event
//! context is asserting on process-global state. See
//! [`pin_callsite_interest_for_test`] for what goes wrong and why holding its
//! guard fixes it.
/// Stop a sibling test thread from poisoning tracing's process-global callsite
/// interest cache while this test asserts on span or event context.
///
/// `tracing` caches every callsite's `Interest` in **process-global** state, and
/// the first thread to reach a callsite fixes that value. While at most one
/// dispatcher is registered, tracing-core takes a fast path
/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
/// newly-registered callsite's interest from whichever subscriber is current
/// *on the registering thread*, and registration happens exactly once (guarded
/// by a CAS in `DefaultCallsite::register`).
///
/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
/// reaches a production callsite first, from a thread with no subscriber
/// installed: the interest is derived from that thread's `NoSubscriber` and
/// cached as `Interest::never()` for the whole process. From then on the
/// callsite is dead for *every* later caller, including a test that installed
/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`, and
/// a `warn!`/`info!` event never fires at all.
///
/// Registering a second, inert dispatcher closes both halves of that race:
///
/// * constructing it rebuilds every *already-registered* callsite's interest
/// against the live dispatcher set — which includes the caller's subscriber —
/// repairing whatever a sibling may already have poisoned; and
/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
/// a callsite registered *later* by any thread is resolved against that live
/// set instead of the registering thread's `NoSubscriber`.
///
/// Call this **after** installing the test's subscriber, and hold the returned
/// guard for the rest of the test. Only the `cargo test` fallback needs it:
/// nextest runs each test in its own process, where there is no sibling thread
/// to lose the race to (see `docs/testing/README.md`).
#[must_use = "callsite interest is only pinned while the returned guard is held"]
pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
}
+2 -2
View File
@@ -2,7 +2,7 @@
## MinIO-generated encrypted fixtures
`rustfs/src/storage/minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
`.\rustfs\scripts\minio_fixture_lab\lab.py`.
It currently covers multipart fixtures for:
@@ -20,5 +20,5 @@ Example:
```powershell
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
cargo +1.97.1 test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
```
@@ -4,13 +4,14 @@ use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use super::sse::SseObjectEncryptionResolver;
use super::storage_api::ecstore_test_support::{
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
};
mod storage_api;
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use storage_api::minio_generated_read::{
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
};
use temp_env::async_with_vars;
use tokio::io::{AsyncReadExt, AsyncWrite};
@@ -48,7 +49,7 @@ impl AsyncWrite for VecAsyncWriter {
fn fixture_root() -> PathBuf {
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../crates/rio-v2/tests/fixtures/minio-generated"))
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated"))
}
fn case_dir(case_id: &str) -> PathBuf {
@@ -136,14 +137,12 @@ async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
],
async move {
let resolver = SseObjectEncryptionResolver;
let (mut reader, offset, length) = GetObjectReader::new_with_resolver(
let (mut reader, offset, length) = GetObjectReader::new(
Box::new(Cursor::new(encrypted)),
None,
&object_info,
&ObjectOptions::default(),
&http::HeaderMap::new(),
Some(&resolver),
)
.await
.map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
@@ -220,12 +219,11 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
readers.push(reader);
}
let erasure = Erasure::try_new(
let erasure = Erasure::new(
file_info.erasure.data_blocks,
file_info.erasure.parity_blocks,
file_info.erasure.block_size,
)
.expect("fixture erasure geometry");
);
let mut writer = VecAsyncWriter::default();
let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
if let Some(err) = err {
-59
View File
@@ -356,14 +356,6 @@ impl FileMeta {
.versions
.partition_point(|existing| existing.header.sorts_before(&new_shallow.header));
self.versions.insert(insert_pos, new_shallow);
// `partition_point` only returns the canonical slot when `versions` is
// already canonically ordered, and nothing establishes that: `FileMeta::load`
// replays the on-disk order verbatim, and metadata written before the
// canonical-order fix ordered equal-`mod_time` ties by insertion rather than
// by `sorts_before`. Re-sort so an insert leaves the list canonical either
// way, matching what `set_idx` already does on the replace path. Cheap: after
// a correctly placed insert the slice is a single sorted run.
self.sort_by_mod_time();
Ok(())
// if !ver.valid() {
@@ -1223,57 +1215,6 @@ mod test {
assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header));
}
/// `add_version_filemata` positions an inserted version with
/// `partition_point(sorts_before)`, which only yields the canonical slot when
/// `versions` is already canonically ordered. Nothing establishes that
/// precondition on load: `FileMeta::unmarshal_msg` pushes versions in file
/// order, and metadata written before the canonical-order fix ordered
/// equal-`mod_time` ties by insertion instead of by `sorts_before`. An insert
/// into such a list must still leave it canonical, the same way `set_idx`
/// already re-sorts on the replace path.
#[test]
fn add_version_filemata_canonicalizes_versions_loaded_out_of_order() {
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
let first = version_for_ordering(VersionType::Object, Uuid::from_u128(70), mod_time, 70);
let second = version_for_ordering(VersionType::Object, Uuid::from_u128(80), mod_time, 80);
let (early, late) = if first.header().sorts_before(&second.header()) {
(first, second)
} else {
(second, first)
};
// Persist the pair the wrong way round to emulate a pre-canonical-order
// xl.meta, bypassing add_version_filemata so the bad order really lands on
// the wire.
let mut legacy = FileMeta::new();
legacy
.versions
.push(FileMetaShallowVersion::try_from(late).expect("shallow later version"));
legacy
.versions
.push(FileMetaShallowVersion::try_from(early).expect("shallow earlier version"));
let mut loaded =
FileMeta::load(&legacy.marshal_msg().expect("serialize legacy metadata")).expect("reload legacy metadata");
assert!(
!loaded.versions[0].header.sorts_before(&loaded.versions[1].header),
"fixture must reach add_version_filemata non-canonically ordered"
);
loaded
.add_version_filemata(version_for_ordering(VersionType::Delete, Uuid::from_u128(90), mod_time, 90))
.expect("insert equal-time delete marker");
assert_eq!(loaded.versions.len(), 3);
assert!(
loaded
.versions
.windows(2)
.all(|pair| pair[0].header.sorts_before(&pair[1].header)),
"insert must leave the version list canonically ordered"
);
}
/// Regression for backlog#799 B16: replication reset state must be
/// persisted under both internal prefixes, never as a bare ARN. A bare-ARN
/// key (produced by `ObjectInfo::replication_state`) has no internal prefix,
+1 -4
View File
@@ -26,9 +26,6 @@ documentation = "https://docs.rs/rustfs-heal/latest/rustfs_heal/"
keywords = ["RustFS", "heal", "erasure-coding", "Minio"]
categories = ["web-programming", "development-tools", "filesystem"]
[lints]
workspace = true
[dependencies]
rustfs-config = { workspace = true }
rustfs-concurrency = { workspace = true }
@@ -56,7 +53,7 @@ serial_test = { workspace = true }
tempfile = { workspace = true }
walkdir = { workspace = true }
http = { workspace = true }
temp-env = { workspace = true, features = ["async_closure"] }
temp-env = { workspace = true }
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
[lib]
+1 -1
View File
@@ -482,7 +482,7 @@ impl HealChannelProcessor {
request_id: client_token,
success: false,
data: None,
error: Some(error_text),
error: Some(error_text.clone()),
};
let _ = response_tx.send(Ok(response.clone()));
self.publish_response(response);
+3 -3
View File
@@ -576,7 +576,7 @@ mod tests {
assert_eq!(handler.event_count(), 1);
handler.add_event(event.clone());
handler.add_event(event);
handler.add_event(event.clone());
assert_eq!(handler.event_count(), 3);
}
@@ -593,7 +593,7 @@ mod tests {
handler.add_event(event.clone());
handler.add_event(event.clone());
handler.add_event(event); // Should remove oldest
handler.add_event(event.clone()); // Should remove oldest
assert_eq!(handler.event_count(), 2);
}
@@ -610,7 +610,7 @@ mod tests {
};
handler.add_event(event.clone());
handler.add_event(event);
handler.add_event(event.clone());
let events = handler.get_events();
assert_eq!(events.len(), 2);
+2 -2
View File
@@ -2892,7 +2892,7 @@ fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTa
gauge!(
"rustfs_heal_task_running",
"type" => type_label.to_string(),
"set" => set_label
"set" => set_label.clone()
)
.set(count as f64);
}
@@ -3482,7 +3482,7 @@ mod tests {
);
assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted);
assert_eq!(queue.push(runnable), QueuePushOutcome::Accepted);
assert_eq!(queue.push(runnable.clone()), QueuePushOutcome::Accepted);
let mut running = HashMap::new();
running.insert("pool_0_set_1".to_string(), 1);
@@ -28,7 +28,6 @@ use rustfs_heal::heal::storage::{
};
use serial_test::serial;
use std::{
future::Future,
path::{Path, PathBuf},
sync::Arc,
};
@@ -53,9 +52,15 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
.collect()
}
/// Disable the grace window while the destructive heal decision is evaluated.
async fn with_dangling_grace_disabled<T>(future: impl Future<Output = T>) -> T {
temp_env::async_with_vars([(GRACE_ENV, Some("0"))], future).await
/// Disable the dangling-delete grace window so the destructive path is genuinely
/// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD
/// be dangling-deleted here. With grace at its 1h default the delete path would be
/// masked and the test would prove nothing.
fn disable_dangling_grace() {
// Safe under nextest: each test runs in its own process and is `#[serial]`.
unsafe {
std::env::set_var(GRACE_ENV, "0");
}
}
/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the
@@ -245,6 +250,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn union_meta_lost_data_present_is_repaired_not_destroyed() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
let bucket = "b920-meta-lost";
let object = "obj.bin";
@@ -262,10 +268,10 @@ mod serial_tests {
}
// Heal the version through the real heal storage (Deep).
let (_result, error) =
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
.await
.expect("heal_object call must not itself error");
let (_result, error) = heal_storage
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
.await
.expect("heal_object call must not itself error");
assert!(
error.is_none(),
"recoverable version must heal without error (must NOT be dangling-deleted): {error:?}"
@@ -294,6 +300,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
let bucket = "b920-torn";
let object = "obj.bin";
@@ -311,10 +318,10 @@ mod serial_tests {
// is a real destructive action, not a no-op).
assert_eq!(count_part_files(&object_dir(&disk_paths[0], bucket, object)), 1);
let (_result, error) =
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
.await
.expect("heal_object call must not itself error");
let (_result, error) = heal_storage
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
.await
.expect("heal_object call must not itself error");
// A dangling delete reports the version as gone (FileVersionNotFound),
// proving the destructive path fired for a genuinely torn write.
assert!(error.is_some(), "a torn (< data_blocks) version must NOT be silently treated as healed");
@@ -342,6 +349,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
let bucket = "b920-reconstruct";
let object = "obj.bin";
@@ -365,10 +373,10 @@ mod serial_tests {
"disk-walk must enumerate the reconstructable sub-quorum version"
);
let (_result, error) =
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
.await
.expect("heal_object call must not itself error");
let (_result, error) = heal_storage
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
.await
.expect("heal_object call must not itself error");
assert!(error.is_none(), "reconstructable version must heal cleanly: {error:?}");
// part.* + xl.meta physically restored on the 4 wiped disks.
@@ -455,6 +463,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn offline_disk_during_walk_does_not_dangling_delete() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
let bucket = "b920-offline";
let object = "obj.bin";
@@ -482,7 +491,8 @@ mod serial_tests {
remove: false,
..Default::default()
};
let (_result, error) = with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &normal_opts))
let (_result, error) = heal_storage
.heal_object(bucket, object, Some(&v1), &normal_opts)
.await
.expect("heal_object call must not itself error");
assert!(error.is_none(), "quorum-present object must not be destroyed: {error:?}");
@@ -500,6 +510,7 @@ mod serial_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() {
disable_dangling_grace();
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
let bucket = "b1044-deep-verify";
let object = "obj.bin";
@@ -513,10 +524,10 @@ mod serial_tests {
assert_eq!(count_part_files(&object_dir(disk, bucket, object)), 1, "intact shard must remain present");
}
let (_result, error) =
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
.await
.expect("deep heal_object call must not itself error");
let (_result, error) = heal_storage
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
.await
.expect("deep heal_object call must not itself error");
assert!(error.is_none(), "Deep heal must retain the three intact EC2+2 shards: {error:?}");
assert!(
+1 -3
View File
@@ -390,10 +390,8 @@ mod serial_tests {
// ─── 1️⃣ delete format.json on one disk ──────────────
let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
let failed_disk_path = disk_paths[0].with_extension("failed");
std::fs::rename(&disk_paths[0], &failed_disk_path).expect("failed to detach disk_paths[0]");
std::fs::remove_dir_all(&disk_paths[0]).expect("failed to delete all contents under disk_paths[0]");
std::fs::create_dir_all(&disk_paths[0]).expect("failed to recreate disk_paths[0] directory");
assert!(!format_path.exists(), "replacement disk already contains format.json before heal");
println!("✅ Deleted format.json on disk: {:?}", disk_paths[0]);
let (_format_result, format_error) = heal_storage.heal_format(false).await.expect("failed to run heal_format");
@@ -67,7 +67,6 @@ const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
@@ -160,7 +159,6 @@ pub struct InternodeMetricsSnapshot {
pub operation_write_shutdown_errors_total: u64,
pub signature_v1_fallback_total: u64,
pub body_digest_fallback_total: u64,
pub replay_scope_fallback_total: u64,
pub replay_cache_overflow_total: u64,
}
@@ -182,7 +180,6 @@ pub struct InternodeMetrics {
msgpack_json_decode_error_total: AtomicU64,
signature_v1_fallback_total: AtomicU64,
body_digest_fallback_total: AtomicU64,
replay_scope_fallback_total: AtomicU64,
replay_cache_overflow_total: AtomicU64,
}
@@ -434,13 +431,6 @@ impl InternodeMetrics {
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
}
/// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is
/// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`.
pub fn record_replay_scope_fallback(&self) {
self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1);
}
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
/// full. Overflow fails closed, so a sustained non-zero rate means
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
@@ -498,7 +488,6 @@ impl InternodeMetrics {
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
}
}
@@ -521,7 +510,6 @@ impl InternodeMetrics {
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
}
}
@@ -899,19 +887,6 @@ mod tests {
assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
}
#[test]
fn replay_scope_fallback_counter_updates_snapshot_and_resets() {
let metrics = InternodeMetrics::default();
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
metrics.record_replay_scope_fallback();
metrics.record_replay_scope_fallback();
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 2);
metrics.reset_for_test();
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
}
#[test]
fn cluster_peer_flips_offline_after_threshold_and_back_online() {
// Unique addr keeps this independent of the process-global registry / other tests.
-3
View File
@@ -25,9 +25,6 @@ keywords = ["rustfs", "openstack", "keystone", "authentication", "s3"]
categories = ["authentication", "web-programming"]
authors.workspace = true
[lints]
workspace = true
[dependencies]
tokio = { workspace = true, features = ["rt", "sync"] }
reqwest = { workspace = true, features = ["json"] }
-22
View File
@@ -25,25 +25,3 @@ For local KMS end-to-end tests, keep proxy bypass settings:
NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \
cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1
```
## Local Key Export for SSE-S3 Migration Tests
Use the read-only `local_kms_key_decrypt` example to export an AES-256 Local
KMS key as the base64 value expected by `RUSTFS_SSE_S3_MASTER_KEY`:
```bash
export RUSTFS_KMS_LOCAL_MASTER_KEY='<local-kms-at-rest-master-key>'
export RUSTFS_SSE_S3_MASTER_KEY="$(
cargo run -q -p rustfs-kms --example local_kms_key_decrypt -- \
/absolute/path/to/<key-id>.key
)"
```
For a `plaintext-dev-only` Local KMS key file,
`RUSTFS_KMS_LOCAL_MASTER_KEY` is not required.
The example writes only the base64-encoded 32-byte key to stdout. Diagnostics
go to stderr. Never paste its output into logs, shell history, issue comments,
or committed configuration. The export path must remain read-only and must
reuse `LocalKmsClient` decoding so current Argon2id and legacy key-file
compatibility stay aligned with the backend.
+1 -3
View File
@@ -44,9 +44,7 @@ argon2 = { workspace = true }
chacha20poly1305 = { workspace = true }
rand = { workspace = true, features = ["serde"] }
base64 = { workspace = true }
hex = { workspace = true }
sha2 = { workspace = true }
subtle = { workspace = true }
zeroize = { workspace = true, features = ["derive"] }
# Configuration and storage
@@ -57,7 +55,7 @@ tempfile = { workspace = true }
moka = { workspace = true, features = ["future"] }
# Additional dependencies
md-5 = { workspace = true }
md5 = { workspace = true }
arc-swap = { workspace = true }
rustfs-utils = { workspace = true }
rustfs-security-governance = { workspace = true }
@@ -1,112 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use zeroize::Zeroizing;
const LOCAL_KMS_MASTER_KEY_ENV: &str = "RUSTFS_KMS_LOCAL_MASTER_KEY";
fn usage(program: &str) -> String {
format!(
"Usage: {program} <local-kms-key-file>\n\
Reads {LOCAL_KMS_MASTER_KEY_ENV} when the key file is encrypted.\n\
Writes only the base64-encoded 32-byte key to stdout."
)
}
fn resolve_key_file(path: &Path) -> Result<(PathBuf, String), String> {
let canonical = std::fs::canonicalize(path).map_err(|error| format!("cannot open Local KMS key file: {error}"))?;
if canonical.extension().and_then(|extension| extension.to_str()) != Some("key") {
return Err("Local KMS key file must have a .key extension".to_string());
}
let key_dir = canonical
.parent()
.ok_or_else(|| "Local KMS key file must have a parent directory".to_string())?
.to_path_buf();
let key_id = canonical
.file_stem()
.and_then(|stem| stem.to_str())
.filter(|stem| !stem.is_empty())
.ok_or_else(|| "Local KMS key file name must contain a valid UTF-8 key ID".to_string())?
.to_string();
Ok((key_dir, key_id))
}
async fn run() -> Result<(), String> {
let mut args = std::env::args();
let program = args.next().unwrap_or_else(|| "local_kms_key_decrypt".to_string());
let Some(key_file) = args.next() else {
return Err(usage(&program));
};
if args.next().is_some() {
return Err(usage(&program));
}
let (key_dir, key_id) = resolve_key_file(Path::new(&key_file))?;
let master_key = std::env::var(LOCAL_KMS_MASTER_KEY_ENV).ok().filter(|value| !value.is_empty());
let client = LocalKmsClient::new_for_key_export(LocalConfig {
key_dir,
master_key,
file_permissions: Some(0o600),
})
.await
.map_err(|error| error.to_string())?;
let key_material = client
.decrypt_key_material_for_export(&key_id)
.await
.map_err(|error| error.to_string())?;
let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref()));
let mut stdout = io::stdout().lock();
writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}"))
}
#[tokio::main]
async fn main() {
if let Err(error) = run().await {
let _ = writeln!(io::stderr().lock(), "local_kms_key_decrypt: {error}");
std::process::exit(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resolve_key_file_extracts_directory_and_key_id() {
let directory = tempfile::tempdir().expect("create temporary directory");
let key_file = directory.path().join("migration-key.key");
std::fs::write(&key_file, b"{}").expect("create key file");
let (key_dir, key_id) = resolve_key_file(&key_file).expect("resolve key file");
assert_eq!(key_dir, directory.path().canonicalize().expect("canonical directory"));
assert_eq!(key_id, "migration-key");
}
#[test]
fn resolve_key_file_rejects_non_key_extension() {
let directory = tempfile::tempdir().expect("create temporary directory");
let key_file = directory.path().join("migration-key.json");
std::fs::write(&key_file, b"{}").expect("create key file");
let error = resolve_key_file(&key_file).expect_err("non-key file must be rejected");
assert!(error.contains(".key"));
}
}
+124 -307
View File
@@ -35,8 +35,8 @@ use std::collections::HashMap;
use std::path::{Component, Path, PathBuf};
use std::time::Duration;
use tokio::fs;
use tokio::sync::RwLock;
use tracing::{debug, warn};
use zeroize::Zeroizing;
/// Reject key identifiers that would not name a single file directly inside the key
/// directory.
@@ -77,6 +77,8 @@ const LOCAL_KMS_ARGON2_P_COST: u32 = 1;
/// Local KMS client that stores keys in local files
pub struct LocalKmsClient {
config: LocalConfig,
/// In-memory cache of loaded keys for performance
key_cache: RwLock<HashMap<String, MasterKeyInfo>>,
/// Master encryption key for encrypting stored keys
master_cipher: Option<Aes256Gcm>,
/// Legacy pre-beta.9 master cipher for reading pre-Argon2 key files
@@ -137,49 +139,9 @@ impl LocalKmsClient {
(None, None)
};
let client = Self {
config,
master_cipher,
legacy_master_cipher,
dek_crypto: AesDekCrypto::new(),
};
client.validate_existing_keys().await?;
Ok(client)
}
/// Open a Local KMS key directory without creating or modifying any files.
///
/// This constructor is restricted to explicit key-export tooling. Normal
/// backend operation must use [`Self::new`].
pub async fn new_for_key_export(config: LocalConfig) -> Result<Self> {
if !fs::try_exists(&config.key_dir).await? {
return Err(KmsError::configuration_error("Local KMS key directory does not exist"));
}
let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key {
let legacy_key = Self::derive_legacy_master_key(master_key)?;
let legacy_master_cipher = Aes256Gcm::new(&legacy_key);
let salt_path = Self::master_key_salt_path(&config);
let master_cipher = if fs::try_exists(&salt_path).await? {
let salt = fs::read(&salt_path).await?;
let salt: [u8; LOCAL_KMS_MASTER_KEY_SALT_LEN] = salt.try_into().map_err(|_| {
KmsError::configuration_error(format!(
"Local KMS master key salt at {} must be exactly {} bytes",
salt_path.display(),
LOCAL_KMS_MASTER_KEY_SALT_LEN
))
})?;
Aes256Gcm::new(&Self::derive_master_key(master_key, &salt)?)
} else {
Aes256Gcm::new(&legacy_key)
};
(Some(master_cipher), Some(legacy_master_cipher))
} else {
(None, None)
};
Ok(Self {
config,
key_cache: RwLock::new(HashMap::new()),
master_cipher,
legacy_master_cipher,
dek_crypto: AesDekCrypto::new(),
@@ -231,35 +193,10 @@ impl LocalKmsClient {
let mut salt = [0u8; LOCAL_KMS_MASTER_KEY_SALT_LEN];
rand::rng().fill(&mut salt[..]);
let temp_path = config
.key_dir
.join(format!("{LOCAL_KMS_MASTER_KEY_SALT_FILE}.tmp-{}", uuid::Uuid::new_v4()));
fs::write(&temp_path, salt).await?;
Self::set_file_permissions(&temp_path, config.file_permissions).await?;
match fs::hard_link(&temp_path, &salt_path).await {
Ok(()) => {
if let Err(error) = fs::remove_file(&temp_path).await {
warn!(path = ?temp_path, %error, "Failed to remove Local KMS salt temporary file");
}
debug!(path = ?salt_path, "Local KMS master key salt created");
Ok(salt)
}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
let _ = fs::remove_file(&temp_path).await;
let bytes = fs::read(&salt_path).await?;
bytes.try_into().map_err(|_| {
KmsError::configuration_error(format!(
"Local KMS master key salt at {} must be exactly {} bytes",
salt_path.display(),
LOCAL_KMS_MASTER_KEY_SALT_LEN
))
})
}
Err(error) => {
let _ = fs::remove_file(&temp_path).await;
Err(error.into())
}
}
fs::write(&salt_path, salt).await?;
Self::set_file_permissions(&salt_path, config.file_permissions).await?;
debug!(path = ?salt_path, "Local KMS master key salt created");
Ok(salt)
}
#[cfg(unix)]
@@ -305,12 +242,6 @@ impl LocalKmsClient {
let content = fs::read(&key_path).await?;
let stored_key: StoredMasterKey = serde_json::from_slice(&content)?;
if stored_key.key_id != key_id {
return Err(KmsError::invalid_key(format!(
"Local KMS key file identity mismatch: expected {key_id:?}, found {:?}",
stored_key.key_id
)));
}
let encrypted_bytes = BASE64
.decode(&stored_key.encrypted_key_material)
@@ -395,43 +326,8 @@ impl LocalKmsClient {
/// Save a master key to disk
async fn save_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
let key_path = self.master_key_path(&master_key.key_id)?;
let content = self.encode_master_key(master_key, key_material)?;
let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
fs::write(&temp_path, &content).await?;
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
fs::rename(&temp_path, &key_path).await?;
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
Ok(())
}
async fn save_new_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
let key_path = self.master_key_path(&master_key.key_id)?;
let content = self.encode_master_key(master_key, key_material)?;
let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
fs::write(&temp_path, &content).await?;
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
match fs::hard_link(&temp_path, &key_path).await {
Ok(()) => {
if let Err(error) = fs::remove_file(&temp_path).await {
warn!(path = ?temp_path, %error, "Failed to remove Local KMS key temporary file");
}
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key created");
Ok(())
}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
let _ = fs::remove_file(&temp_path).await;
Err(KmsError::key_already_exists(&master_key.key_id))
}
Err(error) => {
let _ = fs::remove_file(&temp_path).await;
Err(error.into())
}
}
}
fn encode_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<Vec<u8>> {
// Encrypt key material if master cipher is available
let (encrypted_key_material, nonce, at_rest_protection) = if let Some(ref cipher) = self.master_cipher {
let mut nonce_bytes = [0u8; 12];
rand::rng().fill(&mut nonce_bytes[..]);
@@ -466,7 +362,17 @@ impl LocalKmsClient {
at_rest_protection,
};
serde_json::to_vec_pretty(&stored_key).map_err(Into::into)
let content = serde_json::to_vec_pretty(&stored_key)?;
// Write to temporary file first, then rename for atomicity
let temp_path = key_path.with_extension("tmp");
fs::write(&temp_path, &content).await?;
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
fs::rename(&temp_path, &key_path).await?;
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
Ok(())
}
/// Get the actual key material for a master key
@@ -475,37 +381,6 @@ impl LocalKmsClient {
Ok(key_material)
}
/// Decrypt an AES-256 Local KMS key for explicit migration tooling.
///
/// The returned buffer is zeroized on drop. Callers must treat the value as
/// plaintext key material and avoid logging or persisting it.
pub async fn decrypt_key_material_for_export(&self, key_id: &str) -> Result<Zeroizing<[u8; 32]>> {
let (stored_key, key_material) = self.decode_stored_key(key_id).await?;
if stored_key.algorithm != "AES_256" {
return Err(KmsError::unsupported_algorithm(stored_key.algorithm));
}
let actual = key_material.len();
let key_material = key_material.try_into().map_err(|_| KmsError::invalid_key_size(32, actual))?;
Ok(Zeroizing::new(key_material))
}
async fn validate_existing_keys(&self) -> Result<()> {
let mut entries = fs::read_dir(&self.config.key_dir).await?;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
if path.extension().is_none_or(|extension| extension != "key") {
continue;
}
let key_id = path
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| KmsError::configuration_error("Local KMS key file name must be valid UTF-8"))?;
self.decode_stored_key(key_id).await?;
}
Ok(())
}
/// Encrypt data using a master key
async fn encrypt_with_master_key(&self, key_id: &str, plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
// Load the actual master key material
@@ -638,7 +513,11 @@ impl KmsClient for LocalKmsClient {
let master_key = MasterKeyInfo::new_with_description(key_id.to_string(), algorithm.to_string(), Some(created_by), None);
// Save to disk
self.save_new_master_key(&master_key, &key_material).await?;
self.save_master_key(&master_key, &key_material).await?;
// Cache the key
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
debug!(key_id, "Local KMS master key created");
Ok(master_key)
@@ -647,7 +526,23 @@ impl KmsClient for LocalKmsClient {
async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> {
debug!("Describing key: {}", key_id);
// Check cache first
{
let cache = self.key_cache.read().await;
if let Some(master_key) = cache.get(key_id) {
return Ok(master_key.clone().into());
}
}
// Load from disk
let master_key = self.load_master_key(key_id).await?;
// Update cache
{
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
}
Ok(master_key.into())
}
@@ -707,6 +602,10 @@ impl KmsClient for LocalKmsClient {
let key_material = self.get_key_material(key_id).await?;
self.save_master_key(&master_key, &key_material).await?;
// Update cache
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
debug!(key_id, "Local KMS key enabled");
Ok(())
}
@@ -722,6 +621,10 @@ impl KmsClient for LocalKmsClient {
let key_material = self.get_key_material(key_id).await?;
self.save_master_key(&master_key, &key_material).await?;
// Update cache
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
debug!(key_id, "Local KMS key disabled");
Ok(())
}
@@ -743,6 +646,10 @@ impl KmsClient for LocalKmsClient {
let key_material = self.get_key_material(key_id).await?;
self.save_master_key(&master_key, &key_material).await?;
// Update cache
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
debug!(key_id, "Local KMS key deletion scheduled");
Ok(())
}
@@ -758,17 +665,31 @@ impl KmsClient for LocalKmsClient {
let key_material = self.get_key_material(key_id).await?;
self.save_master_key(&master_key, &key_material).await?;
// Update cache
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
debug!(key_id, "Local KMS key deletion canceled");
Ok(())
}
async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
if !fs::try_exists(self.master_key_path(key_id)?).await? {
return Err(KmsError::key_not_found(key_id));
}
Err(KmsError::invalid_operation(
"Local KMS key rotation is unavailable until historical key versions can be retained",
))
debug!("Rotating key: {}", key_id);
let mut master_key = self.load_master_key(key_id).await?;
master_key.version += 1;
master_key.rotated_at = Some(Zoned::now());
// Generate new key material
let key_material = generate_key_material(&master_key.algorithm)?;
self.save_master_key(&master_key, &key_material).await?;
// Update cache
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
debug!(key_id, "Local KMS key rotated");
Ok(master_key)
}
async fn health_check(&self) -> Result<()> {
@@ -824,6 +745,11 @@ impl KmsBackend for LocalKmsBackend {
async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
let key_id = request.key_name.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
// `save_master_key` writes through a temp file and renames over the destination, so
// creating a key under an existing name would replace its material and silently
// destroy the ability to decrypt everything wrapped under it. The sibling
// `KmsClient::create_key` has always refused this; the backend path did not, and
// this is the path the admin API uses.
if self.client.master_key_path(&key_id)?.exists() {
return Err(KmsError::key_already_exists(&key_id));
}
@@ -841,8 +767,11 @@ impl KmsBackend for LocalKmsBackend {
request.description.clone(),
);
// Save to disk
self.client.save_new_master_key(&master_key, &key_material).await?;
// Save to disk and cache
self.client.save_master_key(&master_key, &key_material).await?;
let mut cache = self.client.key_cache.write().await;
cache.insert(key_id.clone(), master_key.clone());
master_key
};
@@ -959,6 +888,10 @@ impl KmsBackend for LocalKmsBackend {
.await
.map_err(|e| KmsError::internal_error(format!("Failed to delete key file: {e}")))?;
// Remove from cache
let mut cache = self.client.key_cache.write().await;
cache.remove(key_id);
debug!(key_id, "Local KMS key deleted immediately");
// Return success response for immediate deletion
@@ -1002,6 +935,10 @@ impl KmsBackend for LocalKmsBackend {
self.client.save_master_key(&master_key, &existing_key_material).await?;
// Update cache
let mut cache = self.client.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
// Convert master_key to KeyMetadata for response
let key_metadata = KeyMetadata {
key_id: master_key.key_id.clone(),
@@ -1049,6 +986,10 @@ impl KmsBackend for LocalKmsBackend {
self.client.save_master_key(&master_key, &existing_key_material).await?;
// Update cache
let mut cache = self.client.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
// Convert master_key to KeyMetadata for response
let key_metadata = KeyMetadata {
key_id: master_key.key_id.clone(),
@@ -1248,64 +1189,6 @@ mod tests {
.expect("stored encrypted key should deserialize");
assert_eq!(stored.at_rest_protection, StoredKeyProtection::EncryptedMasterKey);
assert_eq!(stored.nonce.len(), 12);
let wrong_master_error = match LocalKmsClient::new(LocalConfig {
key_dir: client.config.key_dir.clone(),
master_key: Some("wrong-master-key".to_string()),
file_permissions: Some(0o600),
})
.await
{
Ok(_) => panic!("wrong master key must fail initialization"),
Err(error) => error,
};
assert!(matches!(wrong_master_error, KmsError::CryptographicError { .. }));
}
#[tokio::test]
async fn key_export_uses_existing_local_decryption_path_without_writing_files() {
let (client, _temp_dir) = create_test_client().await;
let key_id = "export-key";
client
.create_key(key_id, "AES_256", None)
.await
.expect("create encrypted key");
let expected = client.get_key_material(key_id).await.expect("load expected key material");
let salt_path = LocalKmsClient::master_key_salt_path(&client.config);
let salt_before = fs::read(&salt_path).await.expect("read existing salt");
let export_client = LocalKmsClient::new_for_key_export(client.config.clone())
.await
.expect("open read-only export client");
let exported = export_client
.decrypt_key_material_for_export(key_id)
.await
.expect("decrypt key for export");
assert_eq!(exported.as_ref(), expected.as_slice());
assert_eq!(fs::read(&salt_path).await.expect("read unchanged salt"), salt_before);
}
#[tokio::test]
async fn key_export_accepts_plaintext_dev_only_key_without_master_key() {
let (client, _temp_dir) = create_dev_mode_client().await;
let key_id = "plaintext-export-key";
client
.create_key(key_id, "AES_256", None)
.await
.expect("create plaintext-dev-only key");
let expected = client.get_key_material(key_id).await.expect("load expected key material");
let export_client = LocalKmsClient::new_for_key_export(client.config.clone())
.await
.expect("open read-only export client");
let exported = export_client
.decrypt_key_material_for_export(key_id)
.await
.expect("export plaintext-dev-only key");
assert_eq!(exported.as_ref(), expected.as_slice());
assert!(!LocalKmsClient::master_key_salt_path(&client.config).exists());
}
#[tokio::test]
@@ -1345,57 +1228,17 @@ mod tests {
master_key: None,
file_permissions: Some(0o600),
};
let err = match LocalKmsClient::new(config).await {
Ok(_) => panic!("initialization must reject an unreadable encrypted key"),
Err(error) => error,
};
let client_without_master = LocalKmsClient::new(config)
.await
.expect("client without master key should still initialize in dev-mode tests");
let err = client_without_master
.describe_key("encrypted-key", None)
.await
.expect_err("encrypted key should require a master key to read");
assert!(err.to_string().contains("requires a configured master key"));
}
#[tokio::test]
async fn local_key_rotation_is_rejected_without_overwriting_key_material() {
let (client, _temp_dir) = create_test_client().await;
let key_id = "rotation-key";
client.create_key(key_id, "AES_256", None).await.expect("create key");
let original_material = client.get_key_material(key_id).await.expect("load original material");
let error = client
.rotate_key(key_id, None)
.await
.expect_err("rotation must remain unavailable without historical key versions");
assert!(matches!(error, KmsError::InvalidOperation { .. }));
assert_eq!(
client.get_key_material(key_id).await.expect("reload original material"),
original_material
);
}
#[tokio::test]
async fn startup_rejects_key_file_with_mismatched_embedded_id() {
let (client, temp_dir) = create_dev_mode_client().await;
client.create_key("file-name", "AES_256", None).await.expect("create key");
let key_path = client.master_key_path("file-name").expect("valid key id");
let mut stored: serde_json::Value =
serde_json::from_slice(&fs::read(&key_path).await.expect("read key file")).expect("decode key file");
stored["key_id"] = serde_json::json!("embedded-name");
fs::write(&key_path, serde_json::to_vec_pretty(&stored).expect("encode mismatched key"))
.await
.expect("write mismatched key");
let error = match LocalKmsClient::new(LocalConfig {
key_dir: temp_dir.path().to_path_buf(),
master_key: None,
file_permissions: Some(0o600),
})
.await
{
Ok(_) => panic!("mismatched key identity must fail initialization"),
Err(error) => error,
};
assert!(matches!(error, KmsError::InvalidKey { .. }));
}
#[tokio::test]
async fn test_load_master_key_accepts_legacy_rfc3339_timestamp() {
let (client, _temp_dir) = create_dev_mode_client().await;
@@ -1486,6 +1329,17 @@ mod tests {
)
.await
.expect("write beta.5 fixture");
let mut explicit_protection = stored_key.clone();
let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
fs::write(
temp_dir.path().join("beta5-explicit-key.key"),
serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
)
.await
.expect("write explicit-protection fixture");
let client = LocalKmsClient::new(LocalConfig {
key_dir: temp_dir.path().to_path_buf(),
master_key: Some("beta5-test-master-key".to_string()),
@@ -1499,34 +1353,24 @@ mod tests {
.await
.expect("decrypt beta.5 SHA-256 protected key");
assert_eq!(material, vec![0x42; 32]);
let mut explicit_protection = stored_key.clone();
let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
fs::write(
temp_dir.path().join("beta5-explicit-key.key"),
serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
)
.await
.expect("write explicit-protection fixture");
let explicit_error = client
.get_key_material("beta5-explicit-key")
.await
.expect_err("explicit current protection must not fall back to the beta.5 KDF");
assert!(matches!(explicit_error, KmsError::CryptographicError { .. }));
let wrong_key_error = match LocalKmsClient::new(LocalConfig {
let wrong_key_client = LocalKmsClient::new(LocalConfig {
key_dir: temp_dir.path().to_path_buf(),
master_key: Some("wrong-beta5-master-key".to_string()),
file_permissions: Some(0o600),
})
.await
{
Ok(_) => panic!("wrong beta.5 master key must fail initialization"),
Err(error) => error,
};
assert!(matches!(wrong_key_error, KmsError::CryptographicError { .. }));
.expect("initialize local KMS with wrong beta.5 master key");
let error = wrong_key_client
.get_key_material("beta5-key")
.await
.expect_err("wrong beta.5 master key must not decrypt the fixture");
assert!(matches!(error, KmsError::CryptographicError { .. }));
}
/// R03-CAN-072 / R03-CAN-073: key identifiers arrive from request input, so every path
@@ -1585,7 +1429,9 @@ mod tests {
}
}
/// R07-CAN-103: creating a duplicate key must preserve its original material.
/// R07-CAN-103: `save_master_key` writes a temp file and renames over the destination,
/// so creating a key under an existing name would replace its material and silently
/// destroy the ability to decrypt anything wrapped under it.
#[tokio::test]
async fn backend_create_key_refuses_to_replace_existing_key_material() {
let temp_dir = TempDir::new().expect("Failed to create temp dir");
@@ -1623,33 +1469,4 @@ mod tests {
.expect("original key material must survive the refused create");
assert_eq!(original, after, "existing key material must not be replaced");
}
#[tokio::test]
async fn concurrent_backend_create_allows_only_one_writer() {
let temp_dir = TempDir::new().expect("create key directory");
let config = || LocalConfig {
key_dir: temp_dir.path().to_path_buf(),
master_key: Some("test-master-key".to_string()),
file_permissions: Some(0o600),
};
let first = LocalKmsBackend {
client: LocalKmsClient::new(config()).await.expect("create first client"),
};
let second = LocalKmsBackend {
client: LocalKmsClient::new(config()).await.expect("create second client"),
};
let request = || CreateKeyRequest {
key_name: Some("concurrent-key".to_string()),
..Default::default()
};
let (first_result, second_result) = tokio::join!(first.create_key(request()), second.create_key(request()));
assert_ne!(first_result.is_ok(), second_result.is_ok(), "exactly one create must succeed");
let error = first_result
.err()
.or_else(|| second_result.err())
.expect("one create must fail");
assert!(matches!(error, KmsError::KeyAlreadyExists { .. }));
assert_eq!(first.client.get_key_material("concurrent-key").await.expect("load key").len(), 32);
}
}
+3 -3
View File
@@ -89,7 +89,7 @@ pub use error::{KmsError, KmsUnavailableError, Result};
pub use manager::KmsManager;
pub use service::{DataKey, ObjectEncryptionService};
pub use service_manager::{
KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,
KmsServiceManager, KmsServiceStatus, get_global_encryption_service, get_global_kms_service_manager,
init_global_kms_service_manager,
};
pub use types::*;
@@ -177,8 +177,8 @@ mod tests {
let service1 = manager.get_encryption_service().await.expect("Service should be available");
// Reconfigure to new service (zero-downtime)
let mut config2 = config1;
config2.timeout = std::time::Duration::from_secs(45);
let temp_dir2 = TempDir::new().expect("Failed to create temp dir");
let config2 = KmsConfig::local(temp_dir2.path().to_path_buf()).with_insecure_development_defaults();
manager.reconfigure(config2).await.expect("Reconfiguration should succeed");
// Verify version 2
+2 -8
View File
@@ -20,7 +20,6 @@ use crate::manager::KmsManager;
use crate::types::*;
use base64::Engine;
use jiff::Zoned;
use md5::{Digest as Md5Digest, Md5};
use rand::random;
use std::collections::HashMap;
use std::io::Cursor;
@@ -28,12 +27,6 @@ use tokio::io::{AsyncRead, AsyncReadExt};
use tracing::debug;
use zeroize::Zeroize;
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
}
/// Data key for object encryption
/// SECURITY: This struct automatically zeros sensitive key material when dropped
#[derive(Debug, Clone)]
@@ -493,7 +486,8 @@ impl ObjectEncryptionService {
// Validate key MD5 if provided
if let Some(expected_md5) = customer_key_md5 {
let actual_md5_hex = md5_hex(customer_key);
let actual_md5 = md5::compute(customer_key);
let actual_md5_hex = format!("{actual_md5:x}");
if actual_md5_hex != expected_md5.to_lowercase() {
return Err(KmsError::validation_error("Customer key MD5 mismatch"));
}
+89 -461
View File
@@ -20,67 +20,17 @@ use crate::error::{KmsError, Result};
use crate::manager::KmsManager;
use crate::service::ObjectEncryptionService;
use arc_swap::ArcSwap;
use sha2::{Digest, Sha256};
use std::future::Future;
use std::sync::{
Arc, OnceLock,
atomic::{AtomicU64, Ordering},
};
use subtle::ConstantTimeEq;
use tokio::sync::Mutex;
use tokio::sync::{Mutex, RwLock};
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_KMS: &str = "kms";
const LOG_SUBSYSTEM_SERVICE: &str = "service";
const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
fn local_master_key_fingerprint(master_key: Option<&str>) -> [u8; 32] {
let mut digest = Sha256::new();
digest.update([u8::from(master_key.is_some())]);
if let Some(master_key) = master_key {
digest.update(master_key.as_bytes());
}
digest.finalize().into()
}
fn validate_local_transition(current: Option<&KmsConfig>, new: &KmsConfig) -> Result<()> {
let Some(current) = current else {
return Ok(());
};
let BackendConfig::Local(current_local) = &current.backend_config else {
return Ok(());
};
let BackendConfig::Local(new_local) = &new.backend_config else {
return Err(KmsError::configuration_error("Local KMS backend cannot be changed after configuration"));
};
if current_local.key_dir != new_local.key_dir {
return Err(KmsError::configuration_error(
"Local KMS key directory cannot be changed after configuration",
));
}
if current_local.file_permissions != new_local.file_permissions {
return Err(KmsError::configuration_error(
"Local KMS file permissions cannot be changed after configuration",
));
}
if current.allow_insecure_dev_defaults != new.allow_insecure_dev_defaults {
return Err(KmsError::configuration_error(
"Local KMS development mode cannot be changed after configuration",
));
}
let current_master_key = local_master_key_fingerprint(current_local.master_key.as_deref());
let new_master_key = local_master_key_fingerprint(new_local.master_key.as_deref());
if !bool::from(current_master_key.ct_eq(&new_master_key)) {
return Err(KmsError::configuration_error(
"Local KMS master key cannot be changed after configuration",
));
}
Ok(())
}
/// KMS service status
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub enum KmsServiceStatus {
@@ -94,13 +44,6 @@ pub enum KmsServiceStatus {
Error(String),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum KmsStartOutcome {
Started,
Restarted,
AlreadyRunning,
}
/// Service version information for zero-downtime reconfiguration
#[derive(Clone)]
struct ServiceVersion {
@@ -112,17 +55,16 @@ struct ServiceVersion {
manager: Arc<KmsManager>,
}
#[derive(Clone)]
struct RuntimeState {
config: Option<KmsConfig>,
status: KmsServiceStatus,
current_service: Option<ServiceVersion>,
}
/// Dynamic KMS service manager with versioned services for zero-downtime reconfiguration
pub struct KmsServiceManager {
/// Atomically published configuration, status, and current service.
state: ArcSwap<RuntimeState>,
/// Current service version (if running)
/// Uses ArcSwap for atomic, lock-free service switching
/// This allows instant atomic updates without blocking readers
current_service: ArcSwap<Option<ServiceVersion>>,
/// Current configuration
config: Arc<RwLock<Option<KmsConfig>>>,
/// Current status
status: Arc<RwLock<KmsServiceStatus>>,
/// Version counter (monotonically increasing)
version_counter: Arc<AtomicU64>,
/// Mutex to protect lifecycle operations (start, stop, reconfigure)
@@ -134,11 +76,9 @@ impl KmsServiceManager {
/// Create a new KMS service manager (not configured)
pub fn new() -> Self {
Self {
state: ArcSwap::from_pointee(RuntimeState {
config: None,
status: KmsServiceStatus::NotConfigured,
current_service: None,
}),
current_service: ArcSwap::from_pointee(None),
config: Arc::new(RwLock::new(None)),
status: Arc::new(RwLock::new(KmsServiceStatus::NotConfigured)),
version_counter: Arc::new(AtomicU64::new(0)),
lifecycle_mutex: Arc::new(Mutex::new(())),
}
@@ -146,67 +86,39 @@ impl KmsServiceManager {
/// Get current service status
pub async fn get_status(&self) -> KmsServiceStatus {
self.state.load().status.clone()
self.status.read().await.clone()
}
/// Get current configuration (if any)
pub async fn get_config(&self) -> Option<KmsConfig> {
self.state.load().config.clone()
self.config.read().await.clone()
}
/// Get configuration for status and management responses without static key material.
pub async fn get_redacted_config(&self) -> Option<KmsConfig> {
let mut config = self.state.load().config.clone()?;
Self::redact_config(&mut config);
Some(config)
}
/// Get status and redacted configuration from the same published snapshot.
pub async fn get_redacted_state(&self) -> (KmsServiceStatus, Option<KmsConfig>) {
let state = self.state.load();
let mut config = state.config.clone();
if let Some(config) = &mut config {
Self::redact_config(config);
}
(state.status.clone(), config)
}
fn redact_config(config: &mut KmsConfig) {
let mut config = self.config.read().await.clone()?;
if let BackendConfig::Static(static_config) = &mut config.backend_config {
use zeroize::Zeroize;
static_config.secret_key.zeroize();
}
Some(config)
}
/// Configure KMS with new configuration
pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
self.configure_with_persistence(new_config, || async { Ok(()) }).await
}
/// Configure KMS and publish the in-memory state only after persistence succeeds.
///
/// The persistence callback runs under the lifecycle lock and must not call
/// another lifecycle method on this manager.
pub async fn configure_with_persistence<Persist, PersistFuture>(&self, new_config: KmsConfig, persist: Persist) -> Result<()>
where
Persist: FnOnce() -> PersistFuture,
PersistFuture: Future<Output = Result<()>>,
{
new_config.validate()?;
let _guard = self.lifecycle_mutex.lock().await;
let current = self.state.load_full();
validate_local_transition(current.config.as_ref(), &new_config)?;
if current.current_service.is_some() {
return Err(KmsError::configuration_error(
"Cannot configure KMS while it is running; use reconfigure instead",
));
// Update configuration
{
let mut config = self.config.write().await;
*config = Some(new_config.clone());
}
// Update status
{
let mut status = self.status.write().await;
*status = KmsServiceStatus::Configured;
}
persist().await?;
self.state.store(Arc::new(RuntimeState {
config: Some(new_config),
status: KmsServiceStatus::Configured,
current_service: None,
}));
debug!(
event = EVENT_KMS_SERVICE_STATE,
@@ -224,35 +136,19 @@ impl KmsServiceManager {
self.start_internal().await
}
/// Start or restart KMS with the running-state decision serialized with the lifecycle action.
pub async fn start_or_restart(&self, force: bool) -> Result<KmsStartOutcome> {
let _guard = self.lifecycle_mutex.lock().await;
let running = self.state.load().current_service.is_some();
if running && !force {
return Ok(KmsStartOutcome::AlreadyRunning);
}
self.start_internal().await?;
Ok(if running {
KmsStartOutcome::Restarted
} else {
KmsStartOutcome::Started
})
}
/// Internal start implementation (called within lifecycle mutex)
async fn start_internal(&self) -> Result<()> {
let state = self.state.load_full();
let config = match state.config.as_ref() {
Some(config) => config.clone(),
None => {
let err_msg = "Cannot start KMS: no configuration provided";
error!("{}", err_msg);
self.state.store(Arc::new(RuntimeState {
config: None,
status: KmsServiceStatus::Error(err_msg.to_string()),
current_service: None,
}));
return Err(KmsError::configuration_error(err_msg));
let config = {
let config_guard = self.config.read().await;
match config_guard.as_ref() {
Some(config) => config.clone(),
None => {
let err_msg = "Cannot start KMS: no configuration provided";
error!("{}", err_msg);
let mut status = self.status.write().await;
*status = KmsServiceStatus::Error(err_msg.to_string());
return Err(KmsError::configuration_error(err_msg));
}
}
};
@@ -265,9 +161,17 @@ impl KmsServiceManager {
"KMS service starting"
);
match self.create_healthy_service_version(&config).await {
match self.create_service_version(&config).await {
Ok(service_version) => {
self.publish_running(config, service_version);
// Atomically update to new service version (lock-free, instant)
// ArcSwap::store() is a true atomic operation using CAS
self.current_service.store(Arc::new(Some(service_version)));
// Update status
{
let mut status = self.status.write().await;
*status = KmsServiceStatus::Running;
}
debug!(
event = EVENT_KMS_SERVICE_STATE,
@@ -281,24 +185,13 @@ impl KmsServiceManager {
Err(e) => {
let err_msg = format!("Failed to create KMS backend: {e}");
error!("{}", err_msg);
if state.current_service.is_none() {
self.state.store(Arc::new(RuntimeState {
config: state.config.clone(),
status: KmsServiceStatus::Error(err_msg.clone()),
current_service: None,
}));
}
let mut status = self.status.write().await;
*status = KmsServiceStatus::Error(err_msg.clone());
Err(KmsError::backend_error(&err_msg))
}
}
}
/// Replace the running service without exposing a stopped interval.
pub async fn restart(&self) -> Result<()> {
let _guard = self.lifecycle_mutex.lock().await;
self.start_internal().await
}
/// Stop KMS service
///
/// Note: This stops accepting new operations, but existing operations using
@@ -320,16 +213,15 @@ impl KmsServiceManager {
// Atomically clear current service version (lock-free, instant)
// Note: Existing Arc references will keep the service alive until operations complete
let state = self.state.load_full();
self.state.store(Arc::new(RuntimeState {
config: state.config.clone(),
status: if state.config.is_some() {
KmsServiceStatus::Configured
} else {
KmsServiceStatus::NotConfigured
},
current_service: None,
}));
self.current_service.store(Arc::new(None));
// Update status (keep configuration)
{
let mut status = self.status.write().await;
if !matches!(*status, KmsServiceStatus::NotConfigured) {
*status = KmsServiceStatus::Configured;
}
}
debug!(
event = EVENT_KMS_SERVICE_STATE,
@@ -352,22 +244,6 @@ impl KmsServiceManager {
/// This ensures zero downtime during reconfiguration, even for long-running
/// operations like encrypting large files.
pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
self.reconfigure_with_persistence(new_config, || async { Ok(()) }).await
}
/// Reconfigure KMS after the candidate is healthy and persistence succeeds.
///
/// The persistence callback runs under the lifecycle lock and must not call
/// another lifecycle method on this manager.
pub async fn reconfigure_with_persistence<Persist, PersistFuture>(
&self,
new_config: KmsConfig,
persist: Persist,
) -> Result<()>
where
Persist: FnOnce() -> PersistFuture,
PersistFuture: Future<Output = Result<()>>,
{
let _guard = self.lifecycle_mutex.lock().await;
debug!(
@@ -378,18 +254,30 @@ impl KmsServiceManager {
"KMS service reconfiguring"
);
new_config.validate()?;
validate_local_transition(self.state.load().config.as_ref(), &new_config)?;
// Configure with new config
{
let mut config = self.config.write().await;
*config = Some(new_config.clone());
}
// Create new service version without stopping old one
// This allows existing operations to continue while new operations use new service
match self.create_healthy_service_version(&new_config).await {
match self.create_service_version(&new_config).await {
Ok(new_service_version) => {
// Get old version for logging (lock-free read)
let old_version = self.state.load().current_service.as_ref().map(|sv| sv.version);
let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version);
persist().await?;
// Atomically switch to new service version (lock-free, instant CAS operation)
// This is a true atomic operation - no waiting for locks, instant switch
// Old service will be dropped when no more Arc references exist
self.current_service.store(Arc::new(Some(new_service_version.clone())));
self.publish_running(new_config, new_service_version.clone());
// Update status
{
let mut status = self.status.write().await;
*status = KmsServiceStatus::Running;
}
if let Some(old_ver) = old_version {
info!(
@@ -416,6 +304,8 @@ impl KmsServiceManager {
Err(e) => {
let err_msg = format!("Failed to reconfigure KMS: {e}");
error!("{}", err_msg);
let mut status = self.status.write().await;
*status = KmsServiceStatus::Error(err_msg.clone());
Err(KmsError::backend_error(&err_msg))
}
}
@@ -426,7 +316,7 @@ impl KmsServiceManager {
/// Returns the manager from the current service version.
/// Uses lock-free atomic load for optimal performance.
pub async fn get_manager(&self) -> Option<Arc<KmsManager>> {
self.state.load().current_service.as_ref().map(|sv| sv.manager.clone())
self.current_service.load().as_ref().as_ref().map(|sv| sv.manager.clone())
}
/// Get encryption service (if running)
@@ -436,7 +326,7 @@ impl KmsServiceManager {
/// This ensures new operations always use the latest service version,
/// while existing operations continue using their Arc references.
pub async fn get_encryption_service(&self) -> Option<Arc<ObjectEncryptionService>> {
self.state.load().current_service.as_ref().map(|sv| sv.service.clone())
self.current_service.load().as_ref().as_ref().map(|sv| sv.service.clone())
}
/// Get current service version number
@@ -444,16 +334,14 @@ impl KmsServiceManager {
/// Useful for monitoring and debugging.
/// Uses lock-free atomic load.
pub async fn get_service_version(&self) -> Option<u64> {
self.state.load().current_service.as_ref().map(|sv| sv.version)
self.current_service.load().as_ref().as_ref().map(|sv| sv.version)
}
/// Health check for the KMS service
pub async fn health_check(&self) -> Result<bool> {
let checked_state = self.state.load_full();
match checked_state.current_service.as_ref() {
Some(service_version) => {
let manager = service_version.manager.clone();
let checked_version = service_version.version;
let manager = self.get_manager().await;
match manager {
Some(manager) => {
// Perform health check on the backend
match manager.health_check().await {
Ok(healthy) => {
@@ -464,8 +352,9 @@ impl KmsServiceManager {
}
Err(e) => {
error!("KMS health check error: {}", e);
let _guard = self.lifecycle_mutex.lock().await;
self.mark_health_error_if_current(checked_version, &e);
// Update status to error
let mut status = self.status.write().await;
*status = KmsServiceStatus::Error(format!("Health check failed: {e}"));
Err(e)
}
}
@@ -524,33 +413,6 @@ impl KmsServiceManager {
manager: kms_manager,
})
}
async fn create_healthy_service_version(&self, config: &KmsConfig) -> Result<ServiceVersion> {
let service_version = self.create_service_version(config).await?;
if !service_version.manager.health_check().await? {
return Err(KmsError::backend_error("KMS backend health check failed"));
}
Ok(service_version)
}
fn publish_running(&self, config: KmsConfig, service_version: ServiceVersion) {
self.state.store(Arc::new(RuntimeState {
config: Some(config),
status: KmsServiceStatus::Running,
current_service: Some(service_version),
}));
}
fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) {
let current = self.state.load_full();
if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) {
self.state.store(Arc::new(RuntimeState {
config: current.config.clone(),
status: KmsServiceStatus::Error(format!("Health check failed: {error}")),
current_service: current.current_service.clone(),
}));
}
}
}
impl Default for KmsServiceManager {
@@ -583,11 +445,6 @@ pub async fn get_global_encryption_service() -> Option<Arc<ObjectEncryptionServi
#[cfg(test)]
mod tests {
use super::*;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
fn static_config(key_id: &str, fill: u8) -> KmsConfig {
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
}
#[tokio::test]
async fn configure_rejects_insecure_development_defaults_before_state_update() {
@@ -605,6 +462,8 @@ mod tests {
#[tokio::test]
async fn redacted_config_omits_static_key_material() {
use base64::Engine as _;
let manager = KmsServiceManager::new();
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
manager
@@ -618,235 +477,4 @@ mod tests {
};
assert!(static_config.secret_key.is_empty());
}
#[tokio::test]
async fn configure_persistence_failure_leaves_state_unchanged() {
let manager = KmsServiceManager::new();
let result = manager
.configure_with_persistence(static_config("key-a", 0x11), || async { Err(KmsError::backend_error("persist failed")) })
.await;
assert!(result.is_err());
assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured);
assert!(manager.get_config().await.is_none());
assert!(manager.get_encryption_service().await.is_none());
}
#[tokio::test]
async fn configure_rejects_running_service_without_changing_snapshot() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let version = manager.get_service_version().await;
let result = manager.configure(static_config("key-b", 0x22)).await;
assert!(result.is_err());
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
assert_eq!(manager.get_service_version().await, version);
assert_eq!(
manager.get_config().await.and_then(|config| config.default_key_id),
Some("key-a".to_string())
);
}
#[tokio::test]
async fn reconfigure_persistence_failure_keeps_old_running_snapshot() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let old_version = manager.get_service_version().await;
let old_service = manager.get_encryption_service().await.expect("old service");
let result = manager
.reconfigure_with_persistence(static_config("key-b", 0x22), || async {
Err(KmsError::backend_error("persist failed"))
})
.await;
assert!(result.is_err());
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
assert_eq!(manager.get_service_version().await, old_version);
assert_eq!(
manager.get_config().await.and_then(|config| config.default_key_id),
Some("key-a".to_string())
);
assert!(Arc::ptr_eq(
&old_service,
&manager.get_encryption_service().await.expect("old service remains")
));
}
#[tokio::test]
async fn reconfigure_candidate_failure_keeps_old_running_snapshot() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let old_version = manager.get_service_version().await;
let invalid_parent = tempfile::NamedTempFile::new().expect("temporary file");
let invalid_config = KmsConfig::local(invalid_parent.path().join("keys")).with_insecure_development_defaults();
let result = manager.reconfigure(invalid_config).await;
assert!(result.is_err());
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
assert_eq!(manager.get_service_version().await, old_version);
assert_eq!(
manager.get_config().await.and_then(|config| config.default_key_id),
Some("key-a".to_string())
);
}
#[tokio::test]
async fn restart_never_unpublishes_the_running_service() {
let manager = Arc::new(KmsServiceManager::new());
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let old_version = manager.get_service_version().await.expect("old version");
let restarting = {
let manager = manager.clone();
tokio::spawn(async move { manager.restart().await })
};
while !restarting.is_finished() {
assert!(manager.get_encryption_service().await.is_some());
tokio::task::yield_now().await;
}
restarting.await.expect("restart task").expect("restart");
assert!(manager.get_encryption_service().await.is_some());
assert!(manager.get_service_version().await.expect("new version") > old_version);
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
}
#[tokio::test]
async fn start_or_restart_decides_under_the_lifecycle_lock() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
assert_eq!(manager.start_or_restart(false).await.expect("initial start"), KmsStartOutcome::Started);
let first_version = manager.get_service_version().await.expect("first version");
assert_eq!(
manager.start_or_restart(false).await.expect("already running"),
KmsStartOutcome::AlreadyRunning
);
assert_eq!(manager.get_service_version().await, Some(first_version));
assert_eq!(manager.start_or_restart(true).await.expect("forced restart"), KmsStartOutcome::Restarted);
assert!(manager.get_service_version().await.expect("restarted version") > first_version);
}
#[tokio::test]
async fn stale_health_failure_cannot_poison_new_service_status() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let old_version = manager.get_service_version().await.expect("old version");
manager.restart().await.expect("restart");
manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure"));
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
}
#[tokio::test]
async fn forbidden_local_master_key_change_preserves_running_config_and_service() {
use crate::types::{CreateKeyRequest, KeyUsage};
use std::collections::HashMap;
use tempfile::TempDir;
let key_dir = TempDir::new().expect("create local KMS directory");
let config = |master_key: &str| {
let mut config = KmsConfig::local(key_dir.path().to_path_buf());
let BackendConfig::Local(local) = &mut config.backend_config else {
panic!("local constructor must create local backend config");
};
local.master_key = Some(master_key.to_string());
config.allow_insecure_dev_defaults = true;
config
};
let manager = KmsServiceManager::new();
manager
.configure(config("working-master-key"))
.await
.expect("configure local KMS");
manager.start().await.expect("start local KMS");
manager
.get_manager()
.await
.expect("running KMS manager")
.create_key(CreateKeyRequest {
key_name: Some("existing-key".to_string()),
key_usage: KeyUsage::EncryptDecrypt,
description: None,
policy: None,
tags: HashMap::new(),
origin: None,
})
.await
.expect("create encrypted key");
let service_version = manager.get_service_version().await;
let error = manager
.reconfigure(config("wrong-master-key"))
.await
.expect_err("local master key change must be rejected");
assert!(error.to_string().contains("master key cannot be changed"));
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
assert_eq!(manager.get_service_version().await, service_version);
let current = manager.get_config().await.expect("working config must remain");
let BackendConfig::Local(local) = current.backend_config else {
panic!("working config must remain local");
};
assert_eq!(local.master_key.as_deref(), Some("working-master-key"));
}
#[tokio::test]
async fn configure_cannot_replace_existing_local_backend() {
use base64::Engine as _;
use tempfile::TempDir;
let key_dir = TempDir::new().expect("create local KMS directory");
let mut local = KmsConfig::local(key_dir.path().to_path_buf());
local.allow_insecure_dev_defaults = true;
let manager = KmsServiceManager::new();
manager.configure(local.clone()).await.expect("configure local KMS");
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
let error = manager
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
.await
.expect_err("existing local backend must be immutable");
assert!(error.to_string().contains("backend cannot be changed"));
let current = manager.get_config().await.expect("local config must remain");
assert!(matches!(current.backend_config, BackendConfig::Local(_)));
}
#[tokio::test]
async fn reconfigure_allows_safe_local_runtime_settings_only() {
use tempfile::TempDir;
let key_dir = TempDir::new().expect("create local KMS directory");
let mut initial = KmsConfig::local(key_dir.path().to_path_buf());
initial.allow_insecure_dev_defaults = true;
let manager = KmsServiceManager::new();
manager.configure(initial.clone()).await.expect("configure local KMS");
manager.start().await.expect("start local KMS");
let mut updated = initial;
updated.default_key_id = Some("evaluation-key".to_string());
updated.timeout = std::time::Duration::from_secs(45);
updated.enable_cache = false;
manager
.reconfigure(updated.clone())
.await
.expect("update safe local settings");
let current = manager.get_config().await.expect("updated config");
assert_eq!(current.default_key_id, updated.default_key_id);
assert_eq!(current.timeout, updated.timeout);
assert!(!current.enable_cache);
}
}
+6 -309
View File
@@ -545,8 +545,6 @@ pub struct ScannerMetrics {
pub collected_at: DateTime<Utc>,
#[serde(rename = "current_cycle")]
pub current_cycle: u64,
#[serde(rename = "current_cycle_active", default, skip_serializing_if = "Option::is_none")]
pub current_cycle_active: Option<bool>,
#[serde(rename = "current_started")]
pub current_started: DateTime<Utc>,
#[serde(rename = "cycle_complete_times")]
@@ -720,31 +718,7 @@ pub struct ScannerMetrics {
}
impl ScannerMetrics {
pub fn is_current_cycle_active(&self) -> bool {
self.current_cycle_active.unwrap_or(self.current_cycle > 0)
}
pub fn merge(&mut self, other: &Self) {
// Legacy nodes omit the activity field and use a non-zero cycle as
// their active signal. New nodes publish explicit first-cycle and idle
// states, including cycle zero.
let self_cycle_active = self.is_current_cycle_active();
let other_cycle_active = other.is_current_cycle_active();
let self_cycle_authority = (
self_cycle_active,
self.current_cycle,
self.cycles_completed_at.len(),
self.cycles_completed_at.as_slice(),
self.current_started,
);
let other_cycle_authority = (
other_cycle_active,
other.current_cycle,
other.cycles_completed_at.len(),
other.cycles_completed_at.as_slice(),
other.current_started,
);
let other_cycle_is_authoritative = self_cycle_authority < other_cycle_authority;
let other_is_newer = self.collected_at < other.collected_at;
if other_is_newer {
self.collected_at = other.collected_at;
@@ -880,12 +854,15 @@ impl ScannerMetrics {
self.ongoing_buckets = other.ongoing_buckets;
}
if other_cycle_is_authoritative {
if self.current_cycle < other.current_cycle {
self.current_cycle = other.current_cycle;
self.cycles_completed_at = other.cycles_completed_at.clone();
self.current_started = other.current_started;
}
self.current_cycle_active = Some(self_cycle_active || other_cycle_active);
if other.cycles_completed_at.len() > self.cycles_completed_at.len() {
self.cycles_completed_at = other.cycles_completed_at.clone();
}
if !other.life_time_ops.is_empty() && self.life_time_ops.is_empty() {
self.life_time_ops = other.life_time_ops.clone();
@@ -954,13 +931,7 @@ impl Metrics {
if let Some(scanner) = other.scanner.as_ref() {
match self.scanner {
Some(ref mut s_scanner) => s_scanner.merge(scanner),
None => {
let mut scanner = scanner.clone();
if scanner.current_cycle_active.is_none() {
scanner.current_cycle_active = Some(scanner.is_current_cycle_active());
}
self.scanner = Some(scanner);
}
None => self.scanner = Some(scanner.clone()),
}
}
@@ -1418,280 +1389,6 @@ pub struct Operations {
mod tests {
use super::*;
#[test]
fn scanner_metrics_serializes_cycle_active_presence() {
let missing_value = serde_json::to_value(ScannerMetrics::default()).expect("scanner metrics should serialize");
assert!(missing_value.get("current_cycle_active").is_none());
let missing: ScannerMetrics =
serde_json::from_value(missing_value).expect("older scanner metrics without cycle-active should decode");
assert_eq!(missing.current_cycle_active, None);
let explicit_false_value = serde_json::to_value(ScannerMetrics {
current_cycle_active: Some(false),
..Default::default()
})
.expect("scanner metrics with explicit cycle-active should serialize");
assert_eq!(explicit_false_value["current_cycle_active"], serde_json::Value::Bool(false));
let explicit_false: ScannerMetrics =
serde_json::from_value(explicit_false_value).expect("explicit cycle-active should decode");
assert_eq!(explicit_false.current_cycle_active, Some(false));
}
#[test]
fn scanner_metrics_merge_prefers_an_active_first_cycle() {
let collected_at = Utc::now();
let idle_started = collected_at - chrono::Duration::hours(1);
let active_started = collected_at - chrono::Duration::seconds(5);
let mut scanner = ScannerMetrics {
collected_at,
current_cycle: 0,
current_cycle_active: Some(false),
current_started: idle_started,
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle: 0,
current_cycle_active: Some(true),
current_started: active_started,
..Default::default()
});
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_cycle, 0);
assert_eq!(scanner.current_started, active_started);
}
#[test]
fn metrics_merge_preserves_explicit_active_first_cycle() {
let mut aggregated = Metrics::default();
aggregated.merge(&Metrics {
scanner: Some(ScannerMetrics {
current_cycle_active: Some(true),
..Default::default()
}),
..Default::default()
});
let scanner = aggregated.scanner.expect("aggregated scanner metrics");
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_cycle, 0);
}
#[test]
fn scanner_metrics_merge_preserves_legacy_nonzero_active_signal() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics {
collected_at,
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle: 7,
..Default::default()
});
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_cycle, 7);
}
#[test]
fn metrics_merge_normalizes_first_legacy_scanner_snapshot() {
let legacy = Metrics {
scanner: Some(ScannerMetrics {
current_cycle: 7,
..Default::default()
}),
..Default::default()
};
let mut aggregated = Metrics::default();
aggregated.merge(&legacy);
let scanner = aggregated.scanner.expect("aggregated scanner metrics");
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_cycle, 7);
}
#[test]
fn scanner_metrics_merge_preserves_explicit_inactive_nonzero_cycle() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics::default();
scanner.merge(&ScannerMetrics {
collected_at,
current_cycle: 7,
current_cycle_active: Some(false),
..Default::default()
});
assert_eq!(scanner.current_cycle_active, Some(false));
assert_eq!(scanner.current_cycle, 7);
}
#[test]
fn scanner_metrics_merge_cycle_active_is_order_independent() {
let collected_at = Utc::now();
let legacy_active = ScannerMetrics {
collected_at,
current_cycle: 7,
current_started: collected_at - chrono::Duration::seconds(10),
..Default::default()
};
let explicit_idle = ScannerMetrics {
collected_at,
current_cycle: 0,
current_cycle_active: Some(false),
current_started: collected_at - chrono::Duration::hours(1),
..Default::default()
};
let mut legacy_first = legacy_active.clone();
legacy_first.merge(&explicit_idle);
let mut legacy_second = explicit_idle.clone();
legacy_second.merge(&legacy_active);
assert_eq!(legacy_first.current_cycle_active, Some(true));
assert_eq!(legacy_second.current_cycle_active, Some(true));
assert_eq!(legacy_first.current_cycle, 7);
assert_eq!(legacy_second.current_cycle, 7);
let explicit_inactive_nonzero = ScannerMetrics {
collected_at,
current_cycle: 7,
current_cycle_active: Some(false),
..Default::default()
};
let mut inactive_first = explicit_inactive_nonzero.clone();
inactive_first.merge(&explicit_idle);
let mut inactive_second = explicit_idle;
inactive_second.merge(&explicit_inactive_nonzero);
assert_eq!(inactive_first.current_cycle_active, Some(false));
assert_eq!(inactive_second.current_cycle_active, Some(false));
assert_eq!(inactive_first.current_cycle, 7);
assert_eq!(inactive_second.current_cycle, 7);
}
#[test]
fn scanner_metrics_merge_cycle_authority_is_order_independent() {
let collected_at = Utc::now();
let completion = collected_at - chrono::Duration::minutes(1);
let earlier_active = ScannerMetrics {
collected_at,
current_cycle: 7,
current_cycle_active: Some(true),
current_started: collected_at - chrono::Duration::seconds(10),
cycles_completed_at: vec![completion],
..Default::default()
};
let later_active = ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle: 7,
current_cycle_active: Some(true),
current_started: collected_at - chrono::Duration::seconds(5),
cycles_completed_at: vec![completion],
..Default::default()
};
let mut earlier_first = earlier_active.clone();
earlier_first.merge(&later_active);
let mut later_first = later_active.clone();
later_first.merge(&earlier_active);
assert_eq!(earlier_first.current_started, later_active.current_started);
assert_eq!(later_first.current_started, later_active.current_started);
assert_eq!(earlier_first.cycles_completed_at, later_active.cycles_completed_at);
assert_eq!(later_first.cycles_completed_at, later_active.cycles_completed_at);
let stale_idle = ScannerMetrics {
collected_at,
current_cycle_active: Some(false),
current_started: collected_at - chrono::Duration::hours(1),
..Default::default()
};
let completed_idle = ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle_active: Some(false),
current_started: collected_at - chrono::Duration::seconds(5),
cycles_completed_at: vec![completion],
..Default::default()
};
let mut stale_first = stale_idle.clone();
stale_first.merge(&completed_idle);
let mut completed_first = completed_idle.clone();
completed_first.merge(&stale_idle);
assert_eq!(stale_first.current_started, completed_idle.current_started);
assert_eq!(completed_first.current_started, completed_idle.current_started);
assert_eq!(stale_first.cycles_completed_at, completed_idle.cycles_completed_at);
assert_eq!(completed_first.cycles_completed_at, completed_idle.cycles_completed_at);
}
#[test]
fn scanner_metrics_merge_cycle_authority_is_associative() {
let collected_at = Utc::now();
let older_completion = collected_at - chrono::Duration::minutes(3);
let last_completion = collected_at - chrono::Duration::minutes(1);
let cycle_seven = ScannerMetrics {
collected_at,
current_cycle: 7,
current_cycle_active: Some(true),
current_started: collected_at - chrono::Duration::seconds(10),
cycles_completed_at: vec![older_completion, last_completion],
..Default::default()
};
let cycle_eight = ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle: 8,
current_cycle_active: Some(true),
current_started: collected_at - chrono::Duration::seconds(5),
cycles_completed_at: vec![last_completion],
..Default::default()
};
let newer_idle = ScannerMetrics {
collected_at: collected_at + chrono::Duration::hours(1),
current_cycle_active: Some(false),
current_started: collected_at - chrono::Duration::hours(1),
..Default::default()
};
let mut left_associative = cycle_seven.clone();
left_associative.merge(&cycle_eight);
left_associative.merge(&newer_idle);
let mut right_group = cycle_eight.clone();
right_group.merge(&newer_idle);
let mut right_associative = cycle_seven.clone();
right_associative.merge(&right_group);
assert_eq!(left_associative.current_cycle, 8);
assert_eq!(right_associative.current_cycle, 8);
assert_eq!(left_associative.current_started, cycle_eight.current_started);
assert_eq!(right_associative.current_started, cycle_eight.current_started);
assert_eq!(left_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
assert_eq!(right_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
for order in [
[&cycle_seven, &cycle_eight, &newer_idle],
[&cycle_seven, &newer_idle, &cycle_eight],
[&cycle_eight, &cycle_seven, &newer_idle],
[&cycle_eight, &newer_idle, &cycle_seven],
[&newer_idle, &cycle_seven, &cycle_eight],
[&newer_idle, &cycle_eight, &cycle_seven],
] {
let mut merged = ScannerMetrics::default();
for scanner in order {
merged.merge(scanner);
}
assert_eq!(merged.current_cycle_active, Some(true));
assert_eq!(merged.current_cycle, 8);
assert_eq!(merged.current_started, cycle_eight.current_started);
assert_eq!(merged.cycles_completed_at, cycle_eight.cycles_completed_at);
}
}
#[test]
fn scanner_metrics_merge_aggregates_partial_cycles_by_source() {
let collected_at = Utc::now();
+7 -7
View File
@@ -262,11 +262,11 @@ async fn obs_site_replication_stats() -> ReplicationStats {
}
fn current_scanner_cycle_age_seconds(
current_cycle_active: bool,
current_cycle: u64,
current_started: chrono::DateTime<Utc>,
now: chrono::DateTime<Utc>,
) -> u64 {
if !current_cycle_active {
if current_cycle == 0 {
0
} else {
now.signed_duration_since(current_started).num_seconds().max(0) as u64
@@ -1090,7 +1090,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
let reference_time = metrics.cycles_completed_at.last().copied().unwrap_or(metrics.current_started);
let last_activity_seconds = now.signed_duration_since(reference_time).num_seconds().max(0) as u64;
let active_paths = metrics.active_scan_paths as u64;
let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle_active, metrics.current_started, now);
let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle, metrics.current_started, now);
let current_scan_mode = scanner_scan_mode_code(&metrics.current_scan_mode);
let current_cycle_age = current_cycle_age_seconds as f64;
let last_cycle_duration = metrics.last_cycle_duration_seconds;
@@ -1464,21 +1464,21 @@ mod tests {
fn current_scanner_cycle_age_seconds_returns_zero_when_idle() {
let now = Utc::now();
assert_eq!(current_scanner_cycle_age_seconds(false, now - chrono::Duration::seconds(30), now), 0);
assert_eq!(current_scanner_cycle_age_seconds(0, now - chrono::Duration::seconds(30), now), 0);
}
#[test]
fn current_scanner_cycle_age_seconds_clamps_future_start() {
let now = Utc::now();
assert_eq!(current_scanner_cycle_age_seconds(true, now + chrono::Duration::seconds(30), now), 0);
assert_eq!(current_scanner_cycle_age_seconds(4, now + chrono::Duration::seconds(30), now), 0);
}
#[test]
fn current_scanner_cycle_age_seconds_reports_active_first_cycle_elapsed_time() {
fn current_scanner_cycle_age_seconds_reports_active_elapsed_time() {
let now = Utc::now();
assert_eq!(current_scanner_cycle_age_seconds(true, now - chrono::Duration::seconds(45), now), 45);
assert_eq!(current_scanner_cycle_age_seconds(4, now - chrono::Duration::seconds(45), now), 45);
}
#[test]
+2 -5
View File
@@ -25,9 +25,6 @@ description = "Protocol implementations for RustFS (FTPS, SFTP, etc.)"
keywords = ["ftp", "sftp", "protocol", "storage", "rustfs"]
categories = ["network-programming", "filesystem"]
[lints]
workspace = true
[features]
default = []
ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime", "dep:subtle"]
@@ -47,7 +44,7 @@ swift = [
"dep:tokio-util",
"dep:serde",
"dep:urlencoding",
"dep:md-5",
"dep:md5",
"dep:quick-xml",
"dep:hmac",
"dep:sha1",
@@ -111,7 +108,7 @@ http-body-util = { workspace = true, optional = true }
tokio-util = { workspace = true, optional = true, features = ["rt", "io", "compat"] }
serde = { workspace = true, optional = true, features = ["derive"] }
urlencoding = { workspace = true, optional = true }
md-5 = { workspace = true, optional = true }
md5 = { workspace = true, optional = true }
quick-xml = { workspace = true, optional = true, features = ["serialize"] }
hmac = { workspace = true, optional = true }
sha1 = { workspace = true, optional = true }
+5 -5
View File
@@ -608,7 +608,7 @@ impl StorageBackend for DummyBackend {
inner.put_object_calls.push(PutObjectCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
metadata: input.metadata,
metadata: input.metadata.clone(),
});
let stall = inner.stall_put_object;
let entered = inner.put_object_entered.clone();
@@ -731,7 +731,7 @@ impl StorageBackend for DummyBackend {
inner.create_multipart_calls.push(CreateMultipartCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
metadata: input.metadata,
metadata: input.metadata.clone(),
});
}
match self.inner.lock().expect("lock").create_multipart_upload.pop_front() {
@@ -749,7 +749,7 @@ impl StorageBackend for DummyBackend {
inner.upload_part_calls.push(UploadPartCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
upload_id: input.upload_id,
upload_id: input.upload_id.to_string(),
part_number: input.part_number,
content_length: input.content_length,
});
@@ -787,7 +787,7 @@ impl StorageBackend for DummyBackend {
inner.complete_multipart_calls.push(CompleteCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
upload_id: input.upload_id,
upload_id: input.upload_id.to_string(),
part_count,
});
}
@@ -808,7 +808,7 @@ impl StorageBackend for DummyBackend {
inner.abort_multipart_calls.push(AbortCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
upload_id: input.upload_id,
upload_id: input.upload_id.to_string(),
});
}
match self.inner.lock().expect("lock").abort_multipart_upload.pop_front() {
+3 -1
View File
@@ -379,7 +379,9 @@ where
.await
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
let prefix_with_slash = prefix.clone().map(|p| if p.ends_with('/') { p } else { format!("{}/", p) });
let prefix_with_slash = prefix
.clone()
.map(|p| if p.ends_with('/') { p.to_string() } else { format!("{}/", p) });
let list_input = ListObjectsV2Input::builder()
.bucket(bucket)
+1 -1
View File
@@ -371,7 +371,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
let ftps_user = FtpsUser {
username: principal.username.clone(),
name: identity.credentials.name,
name: identity.credentials.name.clone(),
session_context,
};
+2 -2
View File
@@ -1495,12 +1495,12 @@ mod tests {
let buffer_len_u64 = part_buffer_len as u64;
let phase = match phase_variant {
0 => WritePhase::Buffering {
part_buffer,
part_buffer: part_buffer.clone(),
},
1 => WritePhase::Streaming {
upload_id: "UP-proptest".to_string(),
abort_authorized: true,
part_buffer,
part_buffer: part_buffer.clone(),
uploaded_parts: Vec::new(),
next_part_number,
},
+34 -24
View File
@@ -14,11 +14,11 @@
//! Swift account operations and validation
use super::metadata_update::{ACCOUNT_META_TAG_PREFIX, MetadataUpdate};
use super::storage_api::account::{BucketOperations, MakeBucketOptions};
use super::{SwiftError, SwiftResult};
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, update_swift_bucket_tagging};
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, update_swift_bucket_tagging, validate_metadata};
use rustfs_credentials::Credentials;
use s3s::dto::{Tag, Tagging};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
@@ -131,8 +131,9 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
if let Some(tagging) = &bucket_meta.tagging_config {
for tag in &tagging.tag_set {
if let (Some(key), Some(value)) = (&tag.key, &tag.value)
&& let Some(meta_key) = key.strip_prefix(ACCOUNT_META_TAG_PREFIX)
&& let Some(meta_key) = key.strip_prefix("swift-account-meta-")
{
// Strip "swift-account-meta-" prefix
metadata.insert(meta_key.to_string(), value.clone());
}
}
@@ -146,12 +147,6 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
/// Updates account-level metadata such as TempURL keys.
/// Only updates swift-account-meta-* tags, preserving other tags.
///
/// Swift account POST is additive: `update` names the items to write and the
/// items to drop, and everything else keeps its stored value. Replacing the
/// whole set instead would make an unrelated POST — setting a quota, say —
/// delete the account's TempURL signing key, permanently invalidating every
/// outstanding TempURL and FormPost signature for the account.
///
/// The caller must own the account: this metadata holds the account's TempURL
/// signing key, so writing it for someone else's account would let the writer
/// mint valid pre-signed URLs against that account's objects. Reads
@@ -160,11 +155,11 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
///
/// # Arguments
/// * `account` - Account identifier
/// * `update` - Metadata items to set and to remove (names are prefixed with `swift-account-meta-`)
/// * `metadata` - Metadata key-value pairs to store (keys will be prefixed with `swift-account-meta-`)
/// * `credentials` - Keystone credentials of the caller
pub async fn update_account_metadata(
account: &str,
update: &MetadataUpdate,
metadata: &HashMap<String, String>,
credentials: &Option<Credentials>,
) -> SwiftResult<()> {
let Some(credentials) = credentials.as_ref() else {
@@ -176,15 +171,8 @@ pub async fn update_account_metadata(
// These tags are persisted into the bucket metadata file, which every
// later config write rewrites in full — so unbounded metadata inflates
// the cost of unrelated writes for the life of the account. The item
// count is capped against the merged result, inside the rewrite.
update.validate()?;
// An update that names no item changes nothing, so there is no reason to
// bring the account's metadata bucket into existence for it.
if update.is_empty() {
return Ok(());
}
// the cost of unrelated writes for the life of the account.
validate_metadata(metadata)?;
let bucket_name = get_account_metadata_bucket_name(account);
@@ -202,10 +190,32 @@ pub async fn update_account_metadata(
.map_err(|e| SwiftError::InternalServerError(format!("Failed to create account metadata bucket: {}", e)))?;
}
// Merge into the persisted tags: only the swift-account-meta-* items this
// update names change, and non-Swift tags are left alone. An empty result
// clears the tagging config.
update_swift_bucket_tagging(bucket_name, |current| update.apply_to_tags(current, ACCOUNT_META_TAG_PREFIX)).await
// Rewrite the persisted tags: replace swift-account-meta-* tags with the
// new metadata while preserving other tags. An empty result clears the
// tagging config.
update_swift_bucket_tagging(bucket_name, |current| {
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
tagging.tag_set.retain(|tag| {
if let Some(key) = &tag.key {
!key.starts_with("swift-account-meta-")
} else {
true
}
});
for (key, value) in metadata {
tagging.tag_set.push(Tag {
key: Some(format!("swift-account-meta-{}", key)),
value: Some(value.clone()),
});
}
tagging
})
.await?;
Ok(())
}
/// Get TempURL key for account
+3 -1
View File
@@ -627,7 +627,9 @@ mod tests {
#[test]
fn test_parse_paths_with_empty_lines() {
let body = "/container1/file1.txt\n\n/container2/file2.txt\n \n/container1/file3.txt";
assert_eq!(body.lines().filter(|line| !line.trim().is_empty()).count(), 3);
let paths: Vec<&str> = body.lines().filter(|line| !line.trim().is_empty()).collect();
assert_eq!(paths.len(), 3);
}
/// Tests for the `extract_tar_entries` async function.
+235 -97
View File
@@ -17,13 +17,15 @@
//! This module implements Swift container CRUD operations and container-bucket translation.
use super::account::validate_account_access;
use super::metadata_update::{CONTAINER_META_TAG_PREFIX, MetadataUpdate};
use super::storage_api::container::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations as _, MakeBucketOptions,
};
use super::types::Container;
use super::{SwiftError, SwiftResult};
use super::{get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging};
use super::{
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
validate_metadata,
};
use rustfs_credentials::Credentials;
use s3s::dto::{Tag, Tagging};
use sha2::{Digest, Sha256};
@@ -60,6 +62,30 @@ fn sanitize_storage_error<E: std::fmt::Display>(operation: &str, error: E) -> Sw
SwiftError::InternalServerError(format!("{} operation failed", operation))
}
/// Convert Swift container metadata to S3 tags
///
/// Swift container metadata uses X-Container-Meta-* headers.
/// We store these as S3 tags with "swift-meta-" prefix to distinguish from regular bucket tags.
///
/// Example: X-Container-Meta-Color: Blue → S3 Tag: swift-meta-color=Blue
fn swift_metadata_to_s3_tags(metadata: &std::collections::HashMap<String, String>) -> Option<Tagging> {
let mut tags = Vec::new();
for (key, value) in metadata {
// Store with "swift-meta-" prefix to namespace container metadata
tags.push(Tag {
key: Some(format!("swift-meta-{}", key.to_lowercase())),
value: Some(value.clone()),
});
}
if tags.is_empty() {
None
} else {
Some(Tagging { tag_set: tags })
}
}
/// Convert S3 tags back to Swift container metadata
///
/// Extracts only tags with "swift-meta-" prefix, which represent Swift container metadata.
@@ -68,9 +94,11 @@ fn s3_tags_to_swift_metadata(tagging: &Tagging) -> std::collections::HashMap<Str
let mut metadata = std::collections::HashMap::new();
for tag in &tagging.tag_set {
// Only process tags with "swift-meta-" prefix
if let (Some(key), Some(value)) = (&tag.key, &tag.value)
&& let Some(meta_key) = key.strip_prefix(CONTAINER_META_TAG_PREFIX)
&& let Some(meta_key) = key.strip_prefix("swift-meta-")
{
// Skip "swift-meta-"
metadata.insert(meta_key.to_string(), value.clone());
}
}
@@ -445,15 +473,12 @@ pub async fn get_container_metadata(account: &str, container: &str, credentials:
/// - Returns 204 No Content on success
/// - Returns 404 Not Found if container doesn't exist
/// - Metadata is provided via X-Container-Meta-* headers
/// - The update is additive: items the request does not name keep their stored
/// value, and removal is explicit, via `X-Remove-Container-Meta-{name}` or an
/// empty value
#[allow(dead_code)] // Used by handler
pub async fn update_container_metadata(
account: &str,
container: &str,
credentials: &Credentials,
update: MetadataUpdate,
metadata: std::collections::HashMap<String, String>,
) -> SwiftResult<()> {
// Validate account access and extract project_id
let project_id = validate_account_access(account, credentials)?;
@@ -463,9 +488,8 @@ pub async fn update_container_metadata(
// These tags are persisted into the bucket metadata file, which every
// later config write rewrites in full — so unbounded metadata inflates
// the cost of unrelated writes for the life of the container. The item
// count is capped against the merged result, inside the rewrite.
update.validate()?;
// the cost of unrelated writes for the life of the container.
validate_metadata(&metadata)?;
// Create mapper with default config (tenant prefixing enabled)
let mapper = ContainerMapper::default();
@@ -478,9 +502,7 @@ pub async fn update_container_metadata(
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
// Verify container exists. Checked before the empty-update shortcut below,
// so a POST to a container that does not exist still answers 404 whether
// or not it carried metadata.
// Verify container exists
store
.get_bucket_info(&bucket_name, &BucketOptions::default())
.await
@@ -492,19 +514,32 @@ pub async fn update_container_metadata(
}
})?;
// An update that names no item leaves stored metadata alone, so skip the
// persisted write and the peer reload it triggers rather than rewriting
// the config to its current value. The handler reaches here on every
// container POST, including ACL-only and versioning-only ones.
if update.is_empty() {
return Ok(());
}
// Rewrite the persisted tags: replace swift-meta-* tags with the new
// metadata while preserving non-Swift tags. An empty result clears the
// tagging config.
update_swift_bucket_tagging(bucket_name, |current| {
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
// Merge into the persisted tags: only the swift-meta-* items this update
// names change, so the container's other metadata — and the ACL and
// versioning tags sharing this tag set — survive. An empty result clears
// the tagging config.
update_swift_bucket_tagging(bucket_name, |current| update.apply_to_tags(current, CONTAINER_META_TAG_PREFIX)).await
tagging.tag_set.retain(|tag| {
if let Some(key) = &tag.key {
!key.starts_with("swift-meta-")
} else {
true // Keep tags with no key (shouldn't happen, but be safe)
}
});
if let Some(mut new_tagging) = swift_metadata_to_s3_tags(&metadata) {
tagging.tag_set.append(&mut new_tagging.tag_set);
}
// If metadata.is_empty() and swift_metadata_to_s3_tags returns None,
// we've already removed swift-meta-* tags above, so only non-Swift
// tags remain
tagging
})
.await?;
Ok(())
}
/// Delete a container
@@ -779,7 +814,7 @@ pub async fn enable_versioning(
value: Some(archive_container.to_string()), // Store Swift container name, not S3 bucket name
});
Ok(tagging)
tagging
})
.await?;
@@ -836,7 +871,7 @@ pub async fn disable_versioning(account: &str, container: &str, credentials: &Cr
.tag_set
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
Ok(tagging)
tagging
})
.await?;
@@ -991,7 +1026,7 @@ pub async fn set_container_acl(
});
}
Ok(tagging)
tagging
})
.await?;
@@ -1339,6 +1374,64 @@ mod tests {
assert!(first_char.is_ascii_lowercase() || first_char.is_ascii_digit());
}
#[test]
fn test_swift_metadata_to_s3_tags() {
let mut metadata = std::collections::HashMap::new();
metadata.insert("color".to_string(), "blue".to_string());
metadata.insert("description".to_string(), "test container".to_string());
let tagging = swift_metadata_to_s3_tags(&metadata).unwrap();
assert_eq!(tagging.tag_set.len(), 2);
// Verify tags have swift-meta- prefix
let color_tag = tagging
.tag_set
.iter()
.find(|t| t.key.as_deref() == Some("swift-meta-color"))
.expect("color tag not found");
assert_eq!(color_tag.value.as_deref(), Some("blue"));
let desc_tag = tagging
.tag_set
.iter()
.find(|t| t.key.as_deref() == Some("swift-meta-description"))
.expect("description tag not found");
assert_eq!(desc_tag.value.as_deref(), Some("test container"));
}
#[test]
fn test_swift_metadata_to_s3_tags_empty() {
let metadata = std::collections::HashMap::new();
let tagging = swift_metadata_to_s3_tags(&metadata);
assert!(tagging.is_none());
}
#[test]
fn test_swift_metadata_to_s3_tags_case_normalization() {
let mut metadata = std::collections::HashMap::new();
metadata.insert("Color".to_string(), "Red".to_string());
metadata.insert("PRIORITY".to_string(), "High".to_string());
let tagging = swift_metadata_to_s3_tags(&metadata).unwrap();
// Keys should be lowercased
assert!(tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("swift-meta-color")));
assert!(
tagging
.tag_set
.iter()
.any(|t| t.key.as_deref() == Some("swift-meta-priority"))
);
// Values should be preserved as-is
let color_tag = tagging
.tag_set
.iter()
.find(|t| t.key.as_deref() == Some("swift-meta-color"))
.unwrap();
assert_eq!(color_tag.value.as_deref(), Some("Red"));
}
#[test]
fn test_s3_tags_to_swift_metadata() {
let tagging = Tagging {
@@ -1394,80 +1487,91 @@ mod tests {
#[test]
fn test_metadata_roundtrip() {
// What a POST writes must be what a HEAD reads back: run the items
// through the tag-merge write path and the tag-parse read path.
let update = MetadataUpdate::default()
.set("color", "blue")
.set("owner", "alice")
.set("priority", "high");
// Test that we can convert metadata -> tags -> metadata without loss
let mut original_metadata = std::collections::HashMap::new();
original_metadata.insert("color".to_string(), "blue".to_string());
original_metadata.insert("owner".to_string(), "alice".to_string());
original_metadata.insert("priority".to_string(), "high".to_string());
let tagging = update
.apply_to_tags(None, CONTAINER_META_TAG_PREFIX)
.expect("merge should be accepted");
let recovered = s3_tags_to_swift_metadata(&tagging);
let tagging = swift_metadata_to_s3_tags(&original_metadata).unwrap();
let recovered_metadata = s3_tags_to_swift_metadata(&tagging);
assert_eq!(recovered.len(), 3);
assert_eq!(recovered.get("color").map(String::as_str), Some("blue"));
assert_eq!(recovered.get("owner").map(String::as_str), Some("alice"));
assert_eq!(recovered.get("priority").map(String::as_str), Some("high"));
assert_eq!(recovered_metadata.len(), original_metadata.len());
for (key, value) in &original_metadata {
assert_eq!(
recovered_metadata.get(&key.to_lowercase()),
Some(value),
"Metadata key {} not preserved in roundtrip",
key
);
}
}
#[test]
fn test_tag_preservation_merge_with_existing() {
// A container metadata POST shares its tag set with the container ACL
// and versioning tags, and with whatever S3 tags the bucket carries.
// Only the swift-meta-* items the POST names may change.
let existing = Tagging {
// Test merging Swift metadata with existing non-Swift tags
let mut existing_tagging = Tagging {
tag_set: vec![
Tag {
key: Some("swift-meta-color".to_string()),
value: Some("blue".to_string()),
},
Tag {
key: Some("swift-acl-read".to_string()),
value: Some(".r:*".to_string()),
},
Tag {
key: Some("swift-versions-location".to_string()),
value: Some("archive".to_string()),
},
Tag {
key: Some("env".to_string()),
value: Some("production".to_string()),
},
Tag {
key: Some("team".to_string()),
value: Some("backend".to_string()),
},
],
};
let merged = MetadataUpdate::default()
.set("description", "test")
.apply_to_tags(Some(&existing), CONTAINER_META_TAG_PREFIX)
.expect("merge should be accepted");
let tag = |key: &str| {
merged
.tag_set
.iter()
.find(|t| t.key.as_deref() == Some(key))
.and_then(|t| t.value.as_deref())
};
// Remove old swift-meta-* tags
existing_tagging.tag_set.retain(|tag| {
if let Some(key) = &tag.key {
!key.starts_with("swift-meta-")
} else {
true
}
});
assert_eq!(tag("swift-meta-description"), Some("test"), "the new item should be added");
assert_eq!(tag("swift-meta-color"), Some("blue"), "an unnamed item should be preserved");
assert_eq!(tag("swift-acl-read"), Some(".r:*"), "the ACL tag should be preserved");
assert_eq!(tag("swift-versions-location"), Some("archive"), "the versioning tag should be preserved");
assert_eq!(tag("env"), Some("production"), "non-Swift tags should be preserved");
assert_eq!(merged.tag_set.len(), 5);
// Add new Swift metadata
let mut new_metadata = std::collections::HashMap::new();
new_metadata.insert("description".to_string(), "test".to_string());
let mut new_tagging = swift_metadata_to_s3_tags(&new_metadata).unwrap();
// Merge
existing_tagging.tag_set.append(&mut new_tagging.tag_set);
// Verify: should have env, team, and new swift-meta-description
assert_eq!(existing_tagging.tag_set.len(), 3);
let has_env = existing_tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("env"));
let has_team = existing_tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("team"));
let has_description = existing_tagging
.tag_set
.iter()
.any(|t| t.key.as_deref() == Some("swift-meta-description"));
assert!(has_env, "env tag should be preserved");
assert!(has_team, "team tag should be preserved");
assert!(has_description, "swift-meta-description should be added");
}
#[test]
fn test_tag_preservation_remove_only_swift() {
// Removing the last container metadata item leaves the other tags
// and so must not clear the tagging config.
let existing = Tagging {
// Test that clearing Swift metadata preserves non-Swift tags
let mut existing_tagging = Tagging {
tag_set: vec![
Tag {
key: Some("swift-meta-color".to_string()),
value: Some("blue".to_string()),
},
Tag {
key: Some("swift-meta-owner".to_string()),
value: Some("alice".to_string()),
},
Tag {
key: Some("env".to_string()),
value: Some("production".to_string()),
@@ -1479,28 +1583,37 @@ mod tests {
],
};
let merged = MetadataUpdate::default()
.remove("color")
.apply_to_tags(Some(&existing), CONTAINER_META_TAG_PREFIX)
.expect("merge should be accepted");
// Remove swift-meta-* tags (simulating empty metadata update)
existing_tagging.tag_set.retain(|tag| {
if let Some(key) = &tag.key {
!key.starts_with("swift-meta-")
} else {
true
}
});
assert_eq!(merged.tag_set.len(), 2);
assert!(merged.tag_set.iter().any(|t| t.key.as_deref() == Some("env")));
assert!(merged.tag_set.iter().any(|t| t.key.as_deref() == Some("cost-center")));
assert!(
!merged
.tag_set
.iter()
.any(|t| t.key.as_ref().is_some_and(|k| k.starts_with(CONTAINER_META_TAG_PREFIX))),
"the removed item should be gone"
);
// Verify: should only have env and cost-center
assert_eq!(existing_tagging.tag_set.len(), 2);
let has_env = existing_tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("env"));
let has_cost_center = existing_tagging
.tag_set
.iter()
.any(|t| t.key.as_deref() == Some("cost-center"));
let has_swift_meta = existing_tagging
.tag_set
.iter()
.any(|t| t.key.as_ref().is_some_and(|k| k.starts_with("swift-meta-")));
assert!(has_env, "env tag should be preserved");
assert!(has_cost_center, "cost-center tag should be preserved");
assert!(!has_swift_meta, "all swift-meta-* tags should be removed");
}
#[test]
fn test_tag_preservation_empty_after_swift_removal() {
// Removing every item when nothing else is tagged empties the tag set,
// which is how the caller knows to clear the tagging config.
let existing = Tagging {
// Test that if only Swift tags exist, clearing them results in empty tagging
let mut existing_tagging = Tagging {
tag_set: vec![
Tag {
key: Some("swift-meta-color".to_string()),
@@ -1513,13 +1626,38 @@ mod tests {
],
};
let merged = MetadataUpdate::default()
.remove("color")
.remove("owner")
.apply_to_tags(Some(&existing), CONTAINER_META_TAG_PREFIX)
.expect("merge should be accepted");
// Remove swift-meta-* tags
existing_tagging.tag_set.retain(|tag| {
if let Some(key) = &tag.key {
!key.starts_with("swift-meta-")
} else {
true
}
});
assert!(merged.tag_set.is_empty(), "tagging should be empty after removing every swift-meta-* tag");
// Verify: should be empty
assert!(
existing_tagging.tag_set.is_empty(),
"tagging should be empty after removing all swift-meta-* tags"
);
}
#[test]
fn test_tag_preservation_no_existing_tags() {
// Test adding Swift metadata when no tags exist
let existing_tagging = Tagging { tag_set: vec![] };
let mut new_metadata = std::collections::HashMap::new();
new_metadata.insert("color".to_string(), "blue".to_string());
let mut new_tagging = swift_metadata_to_s3_tags(&new_metadata).unwrap();
let mut merged = existing_tagging.clone();
merged.tag_set.append(&mut new_tagging.tag_set);
// Verify: should have only the new Swift tag
assert_eq!(merged.tag_set.len(), 1);
assert_eq!(merged.tag_set[0].key.as_deref(), Some("swift-meta-color"));
assert_eq!(merged.tag_set[0].value.as_deref(), Some("blue"));
}
// Object Versioning Tests
+35 -17
View File
@@ -20,7 +20,6 @@
use super::container;
use super::dlo;
use super::metadata_update::MetadataUpdate;
use super::object;
use super::slo;
use super::tempurl;
@@ -322,15 +321,32 @@ async fn handle_authenticated_request(
Err(SwiftError::NotImplemented("Swift Account HEAD operation not yet implemented".to_string()))
}
Method::POST => {
// Account metadata update. Additive, per Swift: the request
// names the items to set (X-Account-Meta-*) and the items to
// drop (X-Remove-Account-Meta-*, or an empty value), and
// everything it does not name keeps its stored value. The
// TempURL signing key lives here, so a replacing POST would
// invalidate every outstanding signature for the account.
let update = MetadataUpdate::from_account_headers(&headers);
// Account metadata update - extract headers
let mut metadata = std::collections::HashMap::new();
super::account::update_account_metadata(&account, &update, &credentials_opt).await?;
// Extract X-Account-Meta-* headers
for (key, value) in &headers {
let key_str = key.as_str();
if let Some(meta_key) = key_str.strip_prefix("x-account-meta-") {
// Strip "x-account-meta-"
if let Ok(value_str) = value.to_str() {
metadata.insert(meta_key.to_string(), value_str.to_string());
}
}
}
// Special handling for TempURL key headers
// X-Account-Meta-Temp-URL-Key or X-Account-Meta-Temp-Url-Key
if let Some(tempurl_key) = headers
.get("x-account-meta-temp-url-key")
.or_else(|| headers.get("x-account-meta-temp-Url-key"))
&& let Ok(key_str) = tempurl_key.to_str()
{
metadata.insert("temp-url-key".to_string(), key_str.to_string());
}
// Update account metadata
super::account::update_account_metadata(&account, &metadata, &credentials_opt).await?;
let trans_id = generate_trans_id();
Response::builder()
@@ -565,15 +581,17 @@ async fn handle_authenticated_request(
container::set_container_acl(&account, &container, new_read, new_write, &credentials).await?;
}
// Update container metadata. Additive, per Swift: items the
// request does not name keep their stored value, and removal
// is explicit (X-Remove-Container-Meta-*, or an empty value).
// Still called when the request named no item — an ACL-only
// or versioning-only POST — because this is what reports 404
// for a container that does not exist.
let update = MetadataUpdate::from_container_headers(&headers);
// Update container metadata - now we have access to request headers
let mut metadata = std::collections::HashMap::new();
for (name, value) in headers.iter() {
if let Some(meta_key) = name.as_str().strip_prefix("x-container-meta-")
&& let Ok(value_str) = value.to_str()
{
metadata.insert(meta_key.to_string(), value_str.to_string());
}
}
container::update_container_metadata(&account, &container, &credentials, update).await?;
container::update_container_metadata(&account, &container, &credentials, metadata).await?;
let trans_id = generate_trans_id();
Response::builder()
@@ -1,410 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Account and container metadata updates
//!
//! Swift account and container POSTs are *additive*: an item the request does
//! not mention keeps its stored value, and removal is explicit — either
//! `X-Remove-{Account,Container}-Meta-{name}`, or the item sent with an empty
//! value. Object POST is the one that replaces the whole set; `swift::object`
//! handles that separately and must keep doing so.
//!
//! Getting this wrong is not a cosmetic divergence. Account metadata holds the
//! TempURL signing key, so a POST that set an unrelated item while dropping
//! the rest would invalidate every outstanding TempURL and FormPost signature
//! for the account.
//!
//! This module turns a request's headers into the items to write and the items
//! to drop, and applies that to the bucket tag set the metadata is persisted
//! in.
use super::{MAX_METADATA_COUNT, MAX_METADATA_VALUE_SIZE, SwiftError, SwiftResult};
use axum::http::HeaderMap;
use s3s::dto::{Tag, Tagging};
use std::collections::{BTreeMap, BTreeSet};
/// Request header prefix carrying an account metadata item.
const ACCOUNT_META_HEADER_PREFIX: &str = "x-account-meta-";
/// Request header prefix removing an account metadata item.
const ACCOUNT_META_REMOVE_HEADER_PREFIX: &str = "x-remove-account-meta-";
/// Request header prefix carrying a container metadata item.
const CONTAINER_META_HEADER_PREFIX: &str = "x-container-meta-";
/// Request header prefix removing a container metadata item.
const CONTAINER_META_REMOVE_HEADER_PREFIX: &str = "x-remove-container-meta-";
/// Bucket-tag namespace holding account metadata items.
pub(crate) const ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
/// Bucket-tag namespace holding container metadata items.
///
/// Deliberately narrower than the `swift-` tags around it: the container ACL
/// (`swift-acl-*`) and versioning (`swift-versions-location`) tags share this
/// tag set and must survive a metadata POST.
pub(crate) const CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
/// The metadata changes carried by one account or container POST.
///
/// Item names are held lowercased. Swift metadata names are case-insensitive,
/// HTTP header names arrive lowercased anyway, and a removal has to match the
/// name a previous POST stored — so normalizing once here is what makes
/// `X-Remove-Container-Meta-Color` find a stored `color`.
///
/// Ordered rather than hashed so the persisted tag set — and therefore the
/// serialized XML — comes out in a stable order for a given update.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct MetadataUpdate {
/// Items to write, by name.
items: BTreeMap<String, String>,
/// Names to drop.
removals: BTreeSet<String>,
}
impl MetadataUpdate {
/// Write `name` = `value`.
pub fn set(mut self, name: &str, value: &str) -> Self {
let name = name.to_lowercase();
self.removals.remove(&name);
self.items.insert(name, value.to_string());
self
}
/// Drop `name`, if it is stored.
pub fn remove(mut self, name: &str) -> Self {
let name = name.to_lowercase();
self.items.remove(&name);
self.removals.insert(name);
self
}
/// Parse the metadata headers of an account POST.
pub(crate) fn from_account_headers(headers: &HeaderMap) -> Self {
Self::from_headers(headers, ACCOUNT_META_HEADER_PREFIX, ACCOUNT_META_REMOVE_HEADER_PREFIX)
}
/// Parse the metadata headers of a container POST.
pub(crate) fn from_container_headers(headers: &HeaderMap) -> Self {
Self::from_headers(headers, CONTAINER_META_HEADER_PREFIX, CONTAINER_META_REMOVE_HEADER_PREFIX)
}
/// Parse metadata headers under `item_prefix`, and removals under
/// `remove_prefix`. Both prefixes are lowercase, matching how `http`
/// normalizes header names.
///
/// An item sent with an empty value is a removal — the deletion path
/// Swift clients use when they do not send a dedicated remove header.
/// A name carried by both header forms is removed: the explicit removal
/// wins, as it does in Swift, where a remove header is rewritten into an
/// empty-valued item header.
fn from_headers(headers: &HeaderMap, item_prefix: &str, remove_prefix: &str) -> Self {
let mut update = Self::default();
for (name, value) in headers {
let Some(item) = name.as_str().strip_prefix(item_prefix) else {
continue;
};
// A value that is not valid UTF-8 cannot be stored as a tag.
// Skipping it matches how the rest of the Swift handlers treat
// unreadable header values.
let Ok(value) = value.to_str() else {
continue;
};
update = if value.is_empty() {
update.remove(item)
} else {
update.set(item, value)
};
}
for name in headers.keys() {
if let Some(item) = name.as_str().strip_prefix(remove_prefix) {
update = update.remove(item);
}
}
update
}
/// Whether this update changes anything.
///
/// An ACL-only or versioning-only container POST produces an empty update:
/// it names no metadata item, so it must leave stored metadata alone.
pub(crate) fn is_empty(&self) -> bool {
self.items.is_empty() && self.removals.is_empty()
}
/// Reject oversized values before anything is persisted.
///
/// The item *count* is not checked here — an additive POST has to be
/// measured against the merged result, which only [`Self::apply_to_tags`]
/// can see.
pub(crate) fn validate(&self) -> SwiftResult<()> {
for (name, value) in &self.items {
if value.len() > MAX_METADATA_VALUE_SIZE {
return Err(SwiftError::BadRequest(format!(
"Metadata value for '{}' too large: {} bytes (max: {} bytes)",
name,
value.len(),
MAX_METADATA_VALUE_SIZE
)));
}
}
Ok(())
}
/// Merge this update into the persisted tag set.
///
/// `prefix` is the tag namespace holding the items. Tags outside it — the
/// container ACL and versioning tags, plus any S3 tags the bucket carries
/// — are untouched, and so are the namespaced items this update does not
/// name.
///
/// The item-count cap is checked against the merged result rather than the
/// request: because POSTs are additive, a client could otherwise walk past
/// it one header at a time. This runs inside the bucket metadata write
/// guard, so the count it checks is the one about to be persisted.
pub(crate) fn apply_to_tags(&self, current: Option<&Tagging>, prefix: &str) -> SwiftResult<Tagging> {
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
tagging.tag_set.retain(|tag| match item_name(tag, prefix) {
Some(name) => !self.items.contains_key(name) && !self.removals.contains(name),
None => true,
});
let merged = tagging.tag_set.iter().filter(|tag| item_name(tag, prefix).is_some()).count() + self.items.len();
if merged > MAX_METADATA_COUNT {
return Err(SwiftError::BadRequest(format!(
"Too many metadata headers: {} (max: {})",
merged, MAX_METADATA_COUNT
)));
}
for (name, value) in &self.items {
tagging.tag_set.push(Tag {
key: Some(format!("{}{}", prefix, name)),
value: Some(value.clone()),
});
}
Ok(tagging)
}
}
/// The metadata item name a tag carries, or `None` if the tag does not belong
/// to this namespace.
fn item_name<'a>(tag: &'a Tag, prefix: &str) -> Option<&'a str> {
tag.key.as_deref()?.strip_prefix(prefix)
}
#[cfg(test)]
mod tests {
use super::*;
use axum::http::{HeaderName, HeaderValue};
fn headers(pairs: &[(&str, &str)]) -> HeaderMap {
let mut map = HeaderMap::new();
for (name, value) in pairs {
map.insert(
HeaderName::from_bytes(name.as_bytes()).expect("test header name should parse"),
HeaderValue::from_str(value).expect("test header value should parse"),
);
}
map
}
fn tags(pairs: &[(&str, &str)]) -> Tagging {
Tagging {
tag_set: pairs
.iter()
.map(|(key, value)| Tag {
key: Some((*key).to_string()),
value: Some((*value).to_string()),
})
.collect(),
}
}
fn tag_value<'a>(tagging: &'a Tagging, key: &str) -> Option<&'a str> {
tagging
.tag_set
.iter()
.find(|tag| tag.key.as_deref() == Some(key))
.and_then(|tag| tag.value.as_deref())
}
#[test]
fn from_headers_collects_items_and_ignores_unrelated_headers() {
let update = MetadataUpdate::from_container_headers(&headers(&[
("x-container-meta-color", "blue"),
("x-container-read", ".r:*"),
("content-type", "text/plain"),
]));
assert_eq!(update, MetadataUpdate::default().set("color", "blue"));
}
#[test]
fn from_headers_lowercases_item_names() {
// `http` normalizes header names on the way in, so the mixed case a
// client sends is already gone by the time a handler sees it; the
// lowercasing here is what makes a directly built update match.
let update = MetadataUpdate::from_container_headers(&headers(&[("X-Container-Meta-Color", "Blue")]));
assert_eq!(update, MetadataUpdate::default().set("COLOR", "Blue"));
}
#[test]
fn empty_value_is_a_removal() {
let update = MetadataUpdate::from_container_headers(&headers(&[("x-container-meta-color", "")]));
assert_eq!(update, MetadataUpdate::default().remove("color"));
}
#[test]
fn remove_header_drops_the_item() {
let update = MetadataUpdate::from_account_headers(&headers(&[("x-remove-account-meta-temp-url-key", "x")]));
assert_eq!(update, MetadataUpdate::default().remove("temp-url-key"));
}
#[test]
fn remove_header_wins_over_a_value_for_the_same_item() {
let update = MetadataUpdate::from_container_headers(&headers(&[
("x-container-meta-color", "blue"),
("x-remove-container-meta-color", "x"),
]));
assert_eq!(update, MetadataUpdate::default().remove("color"));
}
#[test]
fn a_removal_header_is_not_mistaken_for_an_item() {
// "x-remove-container-meta-color" must not also parse as the item
// "remove-container-meta-color" or similar.
let update = MetadataUpdate::from_container_headers(&headers(&[("x-remove-container-meta-color", "x")]));
assert!(update.items.is_empty(), "a removal header must not set an item");
}
#[test]
fn a_request_with_no_metadata_headers_is_empty() {
assert!(MetadataUpdate::from_container_headers(&headers(&[("x-container-read", ".r:*")])).is_empty());
assert!(MetadataUpdate::default().is_empty());
assert!(!MetadataUpdate::default().remove("color").is_empty());
}
#[test]
fn apply_preserves_items_the_update_does_not_name() {
let current = tags(&[
("swift-meta-color", "blue"),
("swift-meta-season", "summer"),
("swift-acl-read", ".r:*"),
("swift-versions-location", "archive"),
("unrelated-s3-tag", "keep"),
]);
let merged = MetadataUpdate::default()
.set("mood", "calm")
.apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
.expect("merge should be accepted");
assert_eq!(tag_value(&merged, "swift-meta-color"), Some("blue"));
assert_eq!(tag_value(&merged, "swift-meta-season"), Some("summer"));
assert_eq!(tag_value(&merged, "swift-meta-mood"), Some("calm"));
assert_eq!(tag_value(&merged, "swift-acl-read"), Some(".r:*"));
assert_eq!(tag_value(&merged, "swift-versions-location"), Some("archive"));
assert_eq!(tag_value(&merged, "unrelated-s3-tag"), Some("keep"));
}
#[test]
fn apply_overwrites_a_named_item_exactly_once() {
let current = tags(&[("swift-meta-color", "blue")]);
let merged = MetadataUpdate::default()
.set("color", "red")
.apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
.expect("merge should be accepted");
assert_eq!(merged.tag_set.len(), 1, "overwriting must not duplicate the tag");
assert_eq!(tag_value(&merged, "swift-meta-color"), Some("red"));
}
#[test]
fn apply_drops_only_the_removed_item() {
let current = tags(&[
("swift-meta-color", "blue"),
("swift-meta-season", "summer"),
("swift-acl-read", ".r:*"),
]);
let merged = MetadataUpdate::default()
.remove("color")
.apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
.expect("merge should be accepted");
assert_eq!(tag_value(&merged, "swift-meta-color"), None);
assert_eq!(tag_value(&merged, "swift-meta-season"), Some("summer"));
assert_eq!(tag_value(&merged, "swift-acl-read"), Some(".r:*"));
}
#[test]
fn apply_to_an_untagged_bucket_starts_from_nothing() {
let merged = MetadataUpdate::default()
.set("color", "blue")
.apply_to_tags(None, ACCOUNT_META_TAG_PREFIX)
.expect("merge should be accepted");
assert_eq!(tag_value(&merged, "swift-account-meta-color"), Some("blue"));
}
#[test]
fn apply_caps_the_merged_item_count_not_the_request() {
let current = Tagging {
tag_set: (0..MAX_METADATA_COUNT)
.map(|i| Tag {
key: Some(format!("{}item{}", CONTAINER_META_TAG_PREFIX, i)),
value: Some("v".to_string()),
})
.collect(),
};
// One more item than the container already stores: rejected, even
// though the request itself carries a single header.
let err = MetadataUpdate::default()
.set("overflow", "v")
.apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
.expect_err("exceeding the item cap must be rejected");
assert!(matches!(err, SwiftError::BadRequest(_)), "expected BadRequest, got {err:?}");
// Overwriting an item already counted stays at the cap.
MetadataUpdate::default()
.set("item0", "v2")
.apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
.expect("overwriting an existing item must not trip the cap");
}
#[test]
fn validate_rejects_oversized_values() {
let err = MetadataUpdate::default()
.set("color", &"b".repeat(MAX_METADATA_VALUE_SIZE + 1))
.validate()
.expect_err("an oversized value must be rejected");
assert!(matches!(err, SwiftError::BadRequest(_)), "expected BadRequest, got {err:?}");
MetadataUpdate::default()
.set("color", &"b".repeat(MAX_METADATA_VALUE_SIZE))
.validate()
.expect("a value at the limit must be accepted");
}
}
+3 -6
View File
@@ -44,7 +44,6 @@ pub mod expiration;
pub mod expiration_worker;
pub mod formpost;
pub mod handler;
pub mod metadata_update;
pub mod object;
pub mod quota;
pub mod router;
@@ -58,7 +57,6 @@ pub mod types;
pub mod versioning;
pub use errors::{SwiftError, SwiftResult};
pub use metadata_update::MetadataUpdate;
pub use router::{SwiftRoute, SwiftRouter};
/// Maximum number of metadata headers allowed per resource (Swift standard)
@@ -73,10 +71,9 @@ pub(crate) const MAX_METADATA_VALUE_SIZE: usize = 256;
/// - Total number of metadata entries doesn't exceed MAX_METADATA_COUNT
/// - Individual metadata values don't exceed MAX_METADATA_VALUE_SIZE
///
/// This is the object-metadata form, where a POST replaces the whole set and
/// the request is therefore the complete result. Account and container POSTs
/// are additive, so their count has to be measured against the merged state
/// instead — see [`MetadataUpdate::apply_to_tags`].
/// Applies to object, container and account metadata alike: all three are
/// persisted, and container/account metadata additionally lands in the
/// bucket metadata file that every later config write rewrites in full.
///
/// Returns error if limits are exceeded.
pub(crate) fn validate_metadata(metadata: &std::collections::HashMap<String, String>) -> SwiftResult<()> {
+2 -1
View File
@@ -206,9 +206,10 @@ impl ObjectKeyMapper {
#[allow(dead_code)] // Used in: object operations
pub fn normalize_path(object: &str) -> String {
// Split by '/', filter out empty segments (except if it's the end)
let segments: Vec<&str> = object.split('/').collect();
let has_trailing_slash = object.ends_with('/');
let normalized_segments: Vec<&str> = object.split('/').filter(|s| !s.is_empty()).collect();
let normalized_segments: Vec<&str> = segments.into_iter().filter(|s| !s.is_empty()).collect();
let mut result = normalized_segments.join("/");

Some files were not shown because too many files have changed in this diff Show More