mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-30 18:12:14 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 overtrue
|
22045a8541 |
+14
-92
@@ -156,103 +156,16 @@ jobs:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Prepare test evidence
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
{
|
||||
echo "run_id=${GITHUB_RUN_ID}"
|
||||
echo "job=${GITHUB_JOB}"
|
||||
echo "runner=${RUNNER_NAME}"
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/test-and-lint/run-metadata.txt
|
||||
|
||||
# Clippy runs before the test pass: lint failures are the most common
|
||||
# CI-only breakage and should surface in minutes, not after 20+ minutes
|
||||
# of tests.
|
||||
- name: Run clippy lints
|
||||
run: cargo clippy --all-targets -- -D warnings
|
||||
|
||||
- name: Run nextest tests
|
||||
env:
|
||||
# Three concurrent workspace test links saturate the self-hosted
|
||||
# runner's overlay I/O and can wedge Cargo until the 75m timeout.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
- name: Run tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
# Evidence sampler for issue #5394: the post-mortem pgrep below runs
|
||||
# only after `timeout` has already TERM'd the whole cargo process
|
||||
# group, so it cannot name a wedged process. Sample system and
|
||||
# process state every 60s instead; the last samples before the
|
||||
# timeout show what was stuck (rustc, linker, build script, memory
|
||||
# pressure, ...). The log rides along in the existing artifact.
|
||||
(
|
||||
while true; do
|
||||
{
|
||||
echo "=== $(date --utc --iso-8601=seconds)"
|
||||
echo "--- load"; cat /proc/loadavg
|
||||
echo "--- psi"; grep -H . /proc/pressure/* 2>/dev/null || true
|
||||
echo "--- mem"; free -m
|
||||
echo "--- disk"; df -h / /home/runner 2>/dev/null || df -h /
|
||||
echo "--- top-rss"
|
||||
ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss | head -15
|
||||
echo "--- build/test processes"
|
||||
ps -eo pid,ppid,stat,etime,rss,pcpu,args | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true
|
||||
echo "--- d-state (uninterruptible IO)"
|
||||
ps -eo pid,stat,etime,args | awk 'NR > 1 && $2 ~ /D/' || true
|
||||
echo
|
||||
} >> artifacts/test-and-lint/sampler.log 2>&1 || true
|
||||
sleep 60
|
||||
done
|
||||
) &
|
||||
sampler_pid=$!
|
||||
trap 'kill "${sampler_pid}" 2>/dev/null || true' EXIT
|
||||
set +e
|
||||
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
|
||||
cargo nextest run --profile ci --all --exclude e2e_test \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
|
||||
echo "exit_status=${status}"
|
||||
echo "finished_at=$(date --utc --iso-8601=seconds)"
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|nextest|target/.*/deps/' || true
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Run documentation tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
set +e
|
||||
timeout --verbose --signal=TERM --kill-after=30s 15m \
|
||||
cargo test --all --doc \
|
||||
2>&1 | tee artifacts/test-and-lint/doctest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo test --all --doc"
|
||||
echo "exit_status=${status}"
|
||||
echo "finished_at=$(date --utc --iso-8601=seconds)"
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
cargo nextest run --profile ci --all --exclude e2e_test
|
||||
cargo test --all --doc
|
||||
|
||||
# rustfs/backlog#1289: fail if a seed rule's log anchor no longer exists
|
||||
# verbatim in the source tree (log message drifted without updating the
|
||||
@@ -261,6 +174,15 @@ jobs:
|
||||
- name: Check log-analyzer rule anchors
|
||||
run: ./scripts/check_log_analyzer_rules.sh
|
||||
|
||||
- name: Upload test junit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}
|
||||
path: target/nextest/ci/junit.xml
|
||||
retention-days: 3
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Explicit gate for migration-critical suites. These tests already ran in
|
||||
# the full nextest pass above; a single filtered nextest invocation keeps
|
||||
# the named gate without rebuilding or re-running them one package at a time.
|
||||
@@ -406,7 +328,7 @@ jobs:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build debug binary
|
||||
run: cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
run: cargo build -p rustfs --bins
|
||||
|
||||
- name: Upload debug binary
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
@@ -436,7 +358,7 @@ jobs:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build debug binary with rio-v2
|
||||
run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
|
||||
run: cargo build -p rustfs --bins --features rio-v2
|
||||
|
||||
- name: Upload debug binary
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
|
||||
@@ -66,7 +66,7 @@ env:
|
||||
CARGO_TERM_COLOR: always
|
||||
REGISTRY_DOCKERHUB: rustfs/rustfs
|
||||
REGISTRY_GHCR: ghcr.io/${{ github.repository }}
|
||||
REGISTRY_QUAY: quay.io/rustfs/rustfs
|
||||
REGISTRY_QUAY: quay.io/${{ secrets.QUAY_USERNAME }}/rustfs
|
||||
DOCKER_PLATFORMS: linux/amd64,linux/arm64
|
||||
|
||||
jobs:
|
||||
|
||||
Generated
+15
-16
@@ -3671,13 +3671,12 @@ dependencies = [
|
||||
"flatbuffers",
|
||||
"flate2",
|
||||
"futures",
|
||||
"hex",
|
||||
"http 1.4.2",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"local-ip-address",
|
||||
"md-5 0.11.0",
|
||||
"md5",
|
||||
"opentelemetry-proto",
|
||||
"prost 0.14.4",
|
||||
"rand 0.10.2",
|
||||
@@ -5645,9 +5644,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lazy-regex"
|
||||
version = "3.6.1"
|
||||
version = "3.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4994ba703f78b083e2f7946dac9251abd83fd43a0365f030e99b69be5b4b9ef9"
|
||||
checksum = "6bae91019476d3ec7147de9aa291cadb6d870abf2f3015d2da73a90325ac1496"
|
||||
dependencies = [
|
||||
"lazy-regex-proc_macros",
|
||||
"once_cell",
|
||||
@@ -5656,9 +5655,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lazy-regex-proc_macros"
|
||||
version = "3.6.1"
|
||||
version = "3.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd97232314824e6dbef1918a871bb93f51070455e3715bf26e19a6d01aa977a0"
|
||||
checksum = "4de9c1e1439d8b7b3061b2d209809f447ca33241733d9a3c01eabf2dc8d94358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -7237,6 +7236,12 @@ dependencies = [
|
||||
"path-dedot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "path-clean"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
|
||||
|
||||
[[package]]
|
||||
name = "path-dedot"
|
||||
version = "4.0.1"
|
||||
@@ -8885,7 +8890,7 @@ dependencies = [
|
||||
"libmimalloc-sys",
|
||||
"libsystemd",
|
||||
"matchit 0.9.2",
|
||||
"md-5 0.11.0",
|
||||
"md5",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"mimalloc",
|
||||
@@ -9090,6 +9095,7 @@ name = "rustfs-data-usage"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"path-clean",
|
||||
"rmp-serde",
|
||||
"rustfs-filemeta",
|
||||
"serde",
|
||||
@@ -9205,14 +9211,11 @@ dependencies = [
|
||||
"tonic",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-opentelemetry",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"winapi-util",
|
||||
"windows-sys 0.61.2",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
@@ -9446,10 +9449,9 @@ dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.0",
|
||||
"chacha20poly1305",
|
||||
"hex",
|
||||
"insta",
|
||||
"jiff",
|
||||
"md-5 0.11.0",
|
||||
"md5",
|
||||
"moka",
|
||||
"rand 0.10.2",
|
||||
"reqwest",
|
||||
@@ -9458,7 +9460,6 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"subtle",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
@@ -9716,7 +9717,7 @@ dependencies = [
|
||||
"hyper-util",
|
||||
"ipnetwork",
|
||||
"libunftp",
|
||||
"md-5 0.11.0",
|
||||
"md5",
|
||||
"percent-encoding",
|
||||
"proptest",
|
||||
"quick-xml",
|
||||
@@ -10127,7 +10128,6 @@ dependencies = [
|
||||
"bytes",
|
||||
"convert_case 0.11.0",
|
||||
"crc-fast",
|
||||
"criterion",
|
||||
"flate2",
|
||||
"futures",
|
||||
"hex-simd",
|
||||
@@ -11829,7 +11829,6 @@ dependencies = [
|
||||
"futures-util",
|
||||
"libc",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
|
||||
+1
-3
@@ -265,6 +265,7 @@ memmap2 = "0.9.11"
|
||||
lz4 = "1.28.1"
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
md5 = "0.8.1"
|
||||
mime_guess = "2.0.5"
|
||||
moka = { version = "0.12.15" }
|
||||
netif = "0.1.6"
|
||||
@@ -303,7 +304,6 @@ test-case = "3.3.1"
|
||||
thiserror = "2.0.19"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-core = "0.1.36"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = { version = "0.33" }
|
||||
tracing-subscriber = { version = "0.3.23" }
|
||||
@@ -314,9 +314,7 @@ uuid = { version = "1.24.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.18" }
|
||||
zip = "8.6.0"
|
||||
zstd = "0.13.3"
|
||||
|
||||
@@ -25,9 +25,6 @@ keywords = ["checksum-calculation", "verification", "integrity", "authenticity",
|
||||
categories = ["web-programming", "development-tools", "network-programming"]
|
||||
documentation = "https://docs.rs/rustfs-checksums/latest/rustfs_checksum/"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
crc-fast = { workspace = true }
|
||||
|
||||
+33
-145
@@ -1114,8 +1114,6 @@ pub struct ScannerLastMinute {
|
||||
pub struct ScannerMetricsReport {
|
||||
pub collected_at: DateTime<Utc>,
|
||||
pub current_cycle: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_active: bool,
|
||||
pub current_started: DateTime<Utc>,
|
||||
pub cycles_completed_at: Vec<DateTime<Utc>>,
|
||||
pub ongoing_buckets: usize,
|
||||
@@ -2053,7 +2051,7 @@ impl Metrics {
|
||||
pub fn record_scanner_transition_failed(&self, count: u64) {
|
||||
self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
|
||||
self.record_scanner_source_failed(ScannerWorkSource::Lifecycle, count);
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
self.record_last_cycle_scanner_source_work(ScannerWorkSource::Lifecycle, ScannerSourceWorkUpdate::failed(count));
|
||||
}
|
||||
}
|
||||
@@ -2338,21 +2336,6 @@ impl Metrics {
|
||||
*self.cycle_info.write().await = cycle;
|
||||
}
|
||||
|
||||
/// Publish a scanner cycle and its work-accounting baseline as one state transition.
|
||||
pub async fn start_scan_cycle_work_with_cycle(&self, cycle: CurrentCycle) -> ScanCycleWorkSnapshot {
|
||||
let mut current_cycle = self.cycle_info.write().await;
|
||||
let snapshot = self.start_scan_cycle_work();
|
||||
*current_cycle = Some(cycle);
|
||||
snapshot
|
||||
}
|
||||
|
||||
/// Publish the completed work snapshot and idle cycle state as one state transition.
|
||||
pub async fn finish_scan_cycle_work_with_cycle(&self, start: ScanCycleWorkSnapshot, cycle: CurrentCycle) {
|
||||
let mut current_cycle = self.cycle_info.write().await;
|
||||
self.finish_scan_cycle_work(start);
|
||||
*current_cycle = Some(cycle);
|
||||
}
|
||||
|
||||
/// Read the current cycle record.
|
||||
pub async fn get_cycle(&self) -> Option<CurrentCycle> {
|
||||
self.cycle_info.read().await.clone()
|
||||
@@ -2481,7 +2464,7 @@ impl Metrics {
|
||||
&self.current_scan_cycle_replication_repair_work_start,
|
||||
&replication_repair_snapshot,
|
||||
);
|
||||
self.current_scan_cycle_work_active.store(true, Ordering::Release);
|
||||
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
|
||||
snapshot
|
||||
}
|
||||
|
||||
@@ -2493,11 +2476,11 @@ impl Metrics {
|
||||
self.record_scan_cycle_work(work);
|
||||
self.record_scan_cycle_source_work(&source_work);
|
||||
self.record_scan_cycle_replication_repair_work(&replication_repair_work);
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Release);
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -2763,41 +2746,13 @@ impl Metrics {
|
||||
pub async fn report(&self) -> ScannerMetricsReport {
|
||||
let mut m = ScannerMetricsReport::default();
|
||||
|
||||
let has_cycle = {
|
||||
let cycle = self.cycle_info.read().await;
|
||||
let has_cycle = if let Some(cycle) = cycle.as_ref() {
|
||||
m.current_cycle = cycle.current;
|
||||
m.cycles_completed_at = cycle.cycle_completed.clone();
|
||||
m.current_started = cycle.started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
|
||||
if m.current_cycle_active {
|
||||
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
|
||||
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
let current_replication_repair_work =
|
||||
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
|
||||
m.current_cycle_objects_scanned = current_work.objects_scanned;
|
||||
m.current_cycle_directories_scanned = current_work.directories_scanned;
|
||||
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
|
||||
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
|
||||
m.current_cycle_yield_events = current_work.yield_events;
|
||||
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
|
||||
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_ilm_actions = current_work.ilm_actions;
|
||||
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
|
||||
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
|
||||
m.current_cycle_heal_objects = current_work.heal_objects;
|
||||
m.current_cycle_replication_checks = current_work.replication_checks;
|
||||
m.current_cycle_usage_saves = current_work.usage_saves;
|
||||
m.current_cycle_source_work = self.scanner_source_work_snapshots(¤t_source_work);
|
||||
m.current_cycle_replication_repair =
|
||||
self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work);
|
||||
}
|
||||
has_cycle
|
||||
let has_cycle = if let Some(cycle) = self.get_cycle().await {
|
||||
m.current_cycle = cycle.current;
|
||||
m.cycles_completed_at = cycle.cycle_completed;
|
||||
m.current_started = cycle.started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
|
||||
@@ -2838,6 +2793,28 @@ impl Metrics {
|
||||
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
|
||||
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
|
||||
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
|
||||
if self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
|
||||
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
let current_replication_repair_work =
|
||||
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
|
||||
m.current_cycle_objects_scanned = current_work.objects_scanned;
|
||||
m.current_cycle_directories_scanned = current_work.directories_scanned;
|
||||
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
|
||||
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
|
||||
m.current_cycle_yield_events = current_work.yield_events;
|
||||
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
|
||||
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_ilm_actions = current_work.ilm_actions;
|
||||
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
|
||||
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
|
||||
m.current_cycle_heal_objects = current_work.heal_objects;
|
||||
m.current_cycle_replication_checks = current_work.replication_checks;
|
||||
m.current_cycle_usage_saves = current_work.usage_saves;
|
||||
m.current_cycle_source_work = self.scanner_source_work_snapshots(¤t_source_work);
|
||||
m.current_cycle_replication_repair = self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work);
|
||||
}
|
||||
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
|
||||
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
|
||||
m.last_cycle_result_code = last_cycle_result as u64;
|
||||
@@ -4165,8 +4142,6 @@ mod tests {
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert!(report.current_cycle_active);
|
||||
assert_eq!(report.current_cycle, 0);
|
||||
assert_eq!(report.current_cycle_objects_scanned, 7);
|
||||
assert_eq!(report.current_cycle_directories_scanned, 3);
|
||||
assert_eq!(report.current_cycle_bucket_drive_scans, 2);
|
||||
@@ -4183,8 +4158,6 @@ mod tests {
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert!(!report.current_cycle_active);
|
||||
assert_eq!(report.current_cycle, 0);
|
||||
assert_eq!(report.current_cycle_objects_scanned, 0);
|
||||
assert_eq!(report.current_cycle_directories_scanned, 0);
|
||||
assert_eq!(report.current_cycle_bucket_drive_scans, 0);
|
||||
@@ -4211,91 +4184,6 @@ mod tests {
|
||||
assert_eq!(report.last_cycle_usage_saves, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scan_cycle_activity_and_cycle_state_publish_together() {
|
||||
let metrics = Metrics::new();
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let active_cycle = CurrentCycle {
|
||||
current: 12,
|
||||
next: 13,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let cycle_state = metrics.cycle_info.read().await;
|
||||
let mut start_transition = Box::pin(metrics.start_scan_cycle_work_with_cycle(active_cycle));
|
||||
let waker = std::task::Waker::noop();
|
||||
let mut context = std::task::Context::from_waker(waker);
|
||||
assert!(start_transition.as_mut().poll(&mut context).is_pending());
|
||||
assert!(!metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
|
||||
drop(cycle_state);
|
||||
|
||||
let start = start_transition.await;
|
||||
let active = metrics.report().await;
|
||||
assert!(active.current_cycle_active);
|
||||
assert_eq!(active.current_cycle, 12);
|
||||
assert_eq!(active.current_started, cycle_started);
|
||||
|
||||
let idle_cycle = CurrentCycle {
|
||||
current: 0,
|
||||
next: 13,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_state = metrics.cycle_info.read().await;
|
||||
let mut finish_transition = Box::pin(metrics.finish_scan_cycle_work_with_cycle(start, idle_cycle));
|
||||
assert!(finish_transition.as_mut().poll(&mut context).is_pending());
|
||||
assert!(metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
|
||||
drop(cycle_state);
|
||||
|
||||
finish_transition.await;
|
||||
let idle = metrics.report().await;
|
||||
assert!(!idle.current_cycle_active);
|
||||
assert_eq!(idle.current_cycle, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_keeps_cycle_identity_and_work_in_one_snapshot() {
|
||||
let metrics = Metrics::new();
|
||||
let cycle_ten = CurrentCycle {
|
||||
current: 10,
|
||||
next: 11,
|
||||
started: Utc::now() - chrono::Duration::seconds(10),
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
|
||||
|
||||
let paths = metrics.current_paths.write().await;
|
||||
let mut report = Box::pin(metrics.report());
|
||||
let waker = std::task::Waker::noop();
|
||||
let mut context = std::task::Context::from_waker(waker);
|
||||
assert!(report.as_mut().poll(&mut context).is_pending());
|
||||
|
||||
metrics
|
||||
.finish_scan_cycle_work_with_cycle(cycle_ten_start, CurrentCycle { current: 0, ..cycle_ten })
|
||||
.await;
|
||||
let cycle_eleven_start = metrics
|
||||
.start_scan_cycle_work_with_cycle(CurrentCycle {
|
||||
current: 11,
|
||||
next: 12,
|
||||
started: Utc::now(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
|
||||
|
||||
drop(paths);
|
||||
let snapshot = report.await;
|
||||
|
||||
assert_eq!(snapshot.current_cycle, 10);
|
||||
assert_eq!(snapshot.current_cycle_objects_scanned, 1);
|
||||
|
||||
metrics
|
||||
.finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_ilm_actions_ignore_global_ilm_work() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -10,9 +10,6 @@ description = "Shared concurrency contract types for RustFS - workload admission
|
||||
keywords = ["rustfs", "concurrency", "admission", "backpressure", "workers"]
|
||||
categories = ["concurrency", "filesystem"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# Internal crates
|
||||
rustfs-io-core = { workspace = true }
|
||||
|
||||
@@ -158,33 +158,17 @@ pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
|
||||
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
|
||||
|
||||
/// Require the replay-scoped internode RPC signature after the fleet has converged on it.
|
||||
///
|
||||
/// The default keeps v1/v2 peers available during a rolling upgrade. Operators may set this only
|
||||
/// after `rustfs_system_network_internode_replay_scope_fallback_total` remains zero for a full
|
||||
/// release window. The node still accepts a v2-authenticated `Ping` carrying an epoch challenge:
|
||||
/// that narrowly scoped bootstrap lets an upgraded client learn the receiving process epoch and
|
||||
/// immediately retry with the replay-scoped signature after a peer restart.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT: &str = "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT: bool = false;
|
||||
|
||||
// Compile-time invariant: mixed-version clusters must remain available until operators make the
|
||||
// observed fallback counter an explicit strictness decision.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
|
||||
|
||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||
/// one-time consumption of authenticated RPC signatures.
|
||||
/// one-time consumption of body-bound v2 signatures.
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
|
||||
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
|
||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
|
||||
/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
|
||||
/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
|
||||
/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
|
||||
/// the shared secret) — and increments
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
|
||||
/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
|
||||
/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
|
||||
/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
|
||||
/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
|
||||
/// shared secret) — and increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
|
||||
/// counter means this capacity is undersized for the node's peak mutation rate.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
||||
|
||||
@@ -370,12 +354,6 @@ mod tests {
|
||||
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_replay_scope_strict_env_name_is_stable() {
|
||||
// The fail-open default invariant is asserted at compile time next to the definition.
|
||||
assert_eq!(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_replay_cache_capacity_defaults_and_env_name() {
|
||||
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
|
||||
|
||||
@@ -116,27 +116,6 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
|
||||
/// Default: bitrot verification is enabled on GetObject reads (do not skip).
|
||||
pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
|
||||
|
||||
/// Request writing the complete remote-tier version state into object metadata.
|
||||
///
|
||||
/// This remains ineffective until
|
||||
/// [`ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED`] is also enabled.
|
||||
pub const ENV_TIER_REMOTE_VERSION_STATE_WRITE: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE";
|
||||
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested fleet-wide confirmation for
|
||||
/// [`ENV_TIER_REMOTE_VERSION_STATE_WRITE`].
|
||||
///
|
||||
/// This flag is an operational contract, not automatic capability discovery.
|
||||
/// Operators may enable it only after every node that can write or read
|
||||
/// transitioned object metadata supports the remote version-state schema and
|
||||
/// semantics. Keeping the confirmation separate makes a single-node request or
|
||||
/// a writer whose local opt-in is removed fail closed.
|
||||
pub const ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||
|
||||
// =============================================================================
|
||||
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
||||
// =============================================================================
|
||||
@@ -638,15 +617,3 @@ pub const ENV_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: &str = "RUSTFS_OBJ
|
||||
|
||||
/// Default read-ahead disable concurrency threshold: 4.
|
||||
pub const DEFAULT_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
mod remote_version_state_tests {
|
||||
#[test]
|
||||
fn remote_version_state_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_TIER_REMOTE_VERSION_STATE_WRITE, "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
path-clean = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
@@ -12,10 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use path_clean::PathClean;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
hash::{DefaultHasher, Hash, Hasher},
|
||||
path::Path,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
@@ -332,7 +334,7 @@ impl<'de> Deserialize<'de> for SizeHistogram {
|
||||
impl SizeHistogram {
|
||||
pub fn add(&mut self, size: u64) {
|
||||
let intervals = [
|
||||
(0, 1024 - 1), // LESS_THAN_1024_B
|
||||
(0, 1024), // LESS_THAN_1024_B
|
||||
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
|
||||
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
|
||||
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
|
||||
@@ -360,7 +362,7 @@ impl SizeHistogram {
|
||||
// the sub-ranges in [1 KiB, 512 KiB).
|
||||
const ONE_MIB: u64 = 1024 * 1024;
|
||||
let intervals = [
|
||||
(0, 1024 - 1), // LESS_THAN_1024_B
|
||||
(0, 1024), // LESS_THAN_1024_B
|
||||
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
|
||||
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
|
||||
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
|
||||
@@ -504,35 +506,8 @@ pub struct ReplicationStats {
|
||||
}
|
||||
|
||||
impl ReplicationStats {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
let Self {
|
||||
pending_size,
|
||||
replicated_size,
|
||||
failed_size,
|
||||
failed_count,
|
||||
pending_count,
|
||||
missed_threshold_size,
|
||||
after_threshold_size,
|
||||
missed_threshold_count,
|
||||
after_threshold_count,
|
||||
replicated_count,
|
||||
} = self;
|
||||
|
||||
*pending_size == 0
|
||||
&& *replicated_size == 0
|
||||
&& *failed_size == 0
|
||||
&& *failed_count == 0
|
||||
&& *pending_count == 0
|
||||
&& *missed_threshold_size == 0
|
||||
&& *after_threshold_size == 0
|
||||
&& *missed_threshold_count == 0
|
||||
&& *after_threshold_count == 0
|
||||
&& *replicated_count == 0
|
||||
}
|
||||
|
||||
#[deprecated(note = "use is_empty instead")]
|
||||
pub fn empty(&self) -> bool {
|
||||
self.is_empty()
|
||||
self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -545,19 +520,16 @@ pub struct ReplicationAllStats {
|
||||
}
|
||||
|
||||
impl ReplicationAllStats {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
let Self {
|
||||
replica_size,
|
||||
replica_count,
|
||||
targets,
|
||||
} = self;
|
||||
|
||||
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
|
||||
}
|
||||
|
||||
#[deprecated(note = "use is_empty instead")]
|
||||
pub fn empty(&self) -> bool {
|
||||
self.is_empty()
|
||||
if self.replica_size != 0 && self.replica_count != 0 {
|
||||
return false;
|
||||
}
|
||||
for v in self.targets.values() {
|
||||
if !v.empty() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -811,7 +783,7 @@ impl DataUsageCache {
|
||||
return Some(root);
|
||||
}
|
||||
let mut flat = self.flatten(&root);
|
||||
if flat.replication_stats.as_ref().is_some_and(ReplicationAllStats::is_empty) {
|
||||
if flat.replication_stats.as_ref().is_some_and(|stats| stats.empty()) {
|
||||
flat.replication_stats = None;
|
||||
}
|
||||
Some(flat)
|
||||
@@ -1108,39 +1080,9 @@ fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String
|
||||
}
|
||||
}
|
||||
|
||||
fn clean_data_usage_path(data: &str) -> String {
|
||||
let rooted = data.starts_with('/');
|
||||
let mut parts = Vec::new();
|
||||
|
||||
for part in data.split('/') {
|
||||
match part {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
if parts.last().is_some_and(|last| *last != "..") {
|
||||
parts.pop();
|
||||
} else if !rooted {
|
||||
parts.push(part);
|
||||
}
|
||||
}
|
||||
_ => parts.push(part),
|
||||
}
|
||||
}
|
||||
|
||||
let clean = parts.join("/");
|
||||
match (rooted, clean.is_empty()) {
|
||||
(true, true) => "/".to_string(),
|
||||
(true, false) => format!("/{clean}"),
|
||||
(false, true) => ".".to_string(),
|
||||
(false, false) => clean,
|
||||
}
|
||||
}
|
||||
|
||||
/// Hash a slash-separated path for data usage caching.
|
||||
///
|
||||
/// Cache identifiers are persisted and exchanged across nodes, so their
|
||||
/// normalization must not depend on the host operating system.
|
||||
/// Hash a path for data usage caching
|
||||
pub fn hash_path(data: &str) -> DataUsageHash {
|
||||
DataUsageHash(clean_data_usage_path(data))
|
||||
DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
@@ -1525,23 +1467,6 @@ mod tests {
|
||||
buckets_count: u64,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
("", "."),
|
||||
(".", "."),
|
||||
("/", "/"),
|
||||
("//bucket///prefix/", "/bucket/prefix"),
|
||||
("bucket/./prefix//object", "bucket/prefix/object"),
|
||||
("bucket/a/../b", "bucket/b"),
|
||||
("../bucket/..", ".."),
|
||||
("/../../bucket", "/bucket"),
|
||||
("bucket\\prefix/object", "bucket\\prefix/object"),
|
||||
] {
|
||||
assert_eq!(hash_path(input).key(), expected, "unexpected portable cache key for {input:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completeness_marker_is_additive_for_legacy_named_readers() {
|
||||
let current = DataUsageInfo {
|
||||
@@ -1646,49 +1571,6 @@ mod tests {
|
||||
assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_classifies_adjacent_boundaries_once() {
|
||||
let cases = [
|
||||
(1023, 0),
|
||||
(1024, 1),
|
||||
(64 * 1024 - 1, 1),
|
||||
(64 * 1024, 2),
|
||||
(256 * 1024 - 1, 2),
|
||||
(256 * 1024, 3),
|
||||
(512 * 1024 - 1, 3),
|
||||
(512 * 1024, 4),
|
||||
(1024 * 1024 - 1, 4),
|
||||
(1024 * 1024, 6),
|
||||
(10 * 1024 * 1024 - 1, 6),
|
||||
(10 * 1024 * 1024, 7),
|
||||
(64 * 1024 * 1024 - 1, 7),
|
||||
(64 * 1024 * 1024, 8),
|
||||
(128 * 1024 * 1024 - 1, 8),
|
||||
(128 * 1024 * 1024, 9),
|
||||
(512 * 1024 * 1024 - 1, 9),
|
||||
(512 * 1024 * 1024, 10),
|
||||
];
|
||||
|
||||
for (size, expected_bucket) in cases {
|
||||
let mut hist = SizeHistogram::default();
|
||||
hist.add(size);
|
||||
|
||||
assert_eq!(hist.0.iter().sum::<u64>(), 1, "size {size} must have exactly one physical bucket");
|
||||
assert_eq!(hist.0[expected_bucket], 1, "size {size} must select the expected bucket");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_1024_bytes_contributes_to_compat_rollup() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
hist.add(1024);
|
||||
|
||||
let map = hist.to_map();
|
||||
assert_eq!(map["LESS_THAN_1024_B"], 0);
|
||||
assert_eq!(map["BETWEEN_1024_B_AND_64_KB"], 1);
|
||||
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
@@ -1700,126 +1582,6 @@ mod tests {
|
||||
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_stats_empty_checks_every_field() {
|
||||
type SetField = fn(&mut ReplicationStats);
|
||||
|
||||
let cases: [(&str, SetField); 10] = [
|
||||
("pending_size", |stats| stats.pending_size = 1),
|
||||
("replicated_size", |stats| stats.replicated_size = 1),
|
||||
("failed_size", |stats| stats.failed_size = 1),
|
||||
("failed_count", |stats| stats.failed_count = 1),
|
||||
("pending_count", |stats| stats.pending_count = 1),
|
||||
("missed_threshold_size", |stats| stats.missed_threshold_size = 1),
|
||||
("after_threshold_size", |stats| stats.after_threshold_size = 1),
|
||||
("missed_threshold_count", |stats| stats.missed_threshold_count = 1),
|
||||
("after_threshold_count", |stats| stats.after_threshold_count = 1),
|
||||
("replicated_count", |stats| stats.replicated_count = 1),
|
||||
];
|
||||
|
||||
assert!(ReplicationStats::default().is_empty());
|
||||
for (field, set_nonzero) in cases {
|
||||
let mut stats = ReplicationStats::default();
|
||||
set_nonzero(&mut stats);
|
||||
assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_all_stats_empty_checks_aggregate_fields_independently() {
|
||||
let cases = [
|
||||
(
|
||||
"replica_size",
|
||||
ReplicationAllStats {
|
||||
replica_size: 1,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"replica_count",
|
||||
ReplicationAllStats {
|
||||
replica_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
];
|
||||
|
||||
assert!(ReplicationAllStats::default().is_empty());
|
||||
for (field, stats) in cases {
|
||||
assert!(!stats.is_empty(), "{field} must make aggregate replication stats non-empty");
|
||||
}
|
||||
|
||||
let empty_targets = ReplicationAllStats {
|
||||
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
|
||||
|
||||
let stats = ReplicationAllStats {
|
||||
targets: HashMap::from([
|
||||
("arn:test:empty".to_string(), ReplicationStats::default()),
|
||||
(
|
||||
"arn:test:non-empty".to_string(),
|
||||
ReplicationStats {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!stats.is_empty(), "a non-empty target must make aggregate replication stats non-empty");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_recursive_prunes_empty_and_preserves_pending_replication_stats() {
|
||||
let root = hash_path("bucket");
|
||||
let child = hash_path("bucket/child");
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace_hashed(&root, &None, &DataUsageEntry::default());
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
replication_stats: Some(ReplicationAllStats::default()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.size_recursive("bucket")
|
||||
.expect("bucket usage should flatten")
|
||||
.replication_stats
|
||||
.is_none()
|
||||
);
|
||||
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:pending".to_string(),
|
||||
ReplicationStats {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let flattened = cache.size_recursive("bucket").expect("bucket usage should flatten");
|
||||
let replication = flattened
|
||||
.replication_stats
|
||||
.expect("pending-only replication stats must survive pruning");
|
||||
|
||||
assert_eq!(replication.targets["arn:test:pending"].pending_count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_usage_cache_merge_adds_missing_child() {
|
||||
let mut base = DataUsageCache::default();
|
||||
|
||||
@@ -70,8 +70,7 @@ walkdir.workspace = true
|
||||
base64 = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
hex = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
prost.workspace = true
|
||||
sha2 = { workspace = true }
|
||||
|
||||
@@ -24,10 +24,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||
use serial_test::serial;
|
||||
use sha2::Sha256;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::info;
|
||||
|
||||
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
||||
@@ -71,9 +70,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn content_md5_base64(body: &[u8]) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(body);
|
||||
let digest = hasher.finalize();
|
||||
let digest = md5::compute(body);
|
||||
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
|
||||
}
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ use hyper::body::Incoming;
|
||||
use hyper::server::conn::http1;
|
||||
use hyper::service::service_fn;
|
||||
use hyper_util::rt::{TokioIo, TokioTimer};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use s3s::access::{S3Access, S3AccessContext};
|
||||
use s3s::auth::SimpleAuth;
|
||||
use s3s::dto::{
|
||||
@@ -828,25 +827,13 @@ fn ensure_body_growth(current: usize, added: usize) -> S3Result {
|
||||
|
||||
async fn md5_digest(body: Bytes, permit: OwnedSemaphorePermit) -> S3Result<([u8; 16], OwnedSemaphorePermit)> {
|
||||
if body.len() < 1024 * 1024 {
|
||||
return Ok((md5_bytes(body), permit));
|
||||
return Ok((md5::compute(body).0, permit));
|
||||
}
|
||||
tokio::task::spawn_blocking(move || (md5_bytes(body), permit))
|
||||
tokio::task::spawn_blocking(move || (md5::compute(body).0, permit))
|
||||
.await
|
||||
.map_err(|error| s3s::s3_error!(InternalError, "MD5 worker failed: {error}"))
|
||||
}
|
||||
|
||||
fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result {
|
||||
let total_bytes = state
|
||||
.total_bytes
|
||||
@@ -1018,7 +1005,7 @@ impl S3 for FakeBackend {
|
||||
Some(value) => value,
|
||||
None => {
|
||||
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
|
||||
hex::encode(digest)
|
||||
format!("{:x}", md5::Digest(digest))
|
||||
}
|
||||
};
|
||||
let version = ObjectVersion {
|
||||
@@ -1221,7 +1208,7 @@ impl S3 for FakeBackend {
|
||||
}
|
||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
|
||||
let e_tag = hex::encode(digest);
|
||||
let e_tag = format!("{:x}", md5::Digest(digest));
|
||||
let mut state = lock(&self.store);
|
||||
let existing_bytes = state
|
||||
.uploads
|
||||
@@ -1349,7 +1336,7 @@ impl S3 for FakeBackend {
|
||||
.collect();
|
||||
let (body, digests, _body_permits) = assemble_multipart(assembly_parts, total_len, _body_permits).await?;
|
||||
let part_count = requested.len();
|
||||
let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{}-{part_count}", md5_hex(digests)));
|
||||
let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{:x}-{part_count}", md5::compute(digests)));
|
||||
let version = ObjectVersion {
|
||||
version_id: upload.version_id.clone(),
|
||||
body,
|
||||
|
||||
@@ -13,9 +13,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Cross-process replay / tamper acceptance for internode NodeService RPC
|
||||
//! signatures (<https://github.com/rustfs/backlog/issues/1327>,
|
||||
//! <https://github.com/rustfs/backlog/issues/1542>).
|
||||
//! Cross-process replay / tamper acceptance for the internode NodeService v2 RPC
|
||||
//! signature (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
//!
|
||||
//! # Why this exists on top of the in-process tests
|
||||
//!
|
||||
@@ -79,8 +78,6 @@
|
||||
//! | mixed version: legacy-only still served, not blocked | [`legacy_only_signature_is_accepted_in_default_posture`] |
|
||||
//! | strict flip closes the signature downgrade | [`signature_strict_rejects_legacy_only_downgrade`] |
|
||||
//! | strict flip closes the body-digest downgrade, incl. v1 | [`body_digest_strict_rejects_digestless_mutation`] |
|
||||
//! | replay scope binds every RPC and rejects restart replay | [`replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e`] |
|
||||
//! | strict replay scope allows only Ping bootstrap before v3 | [`replay_scope_strict_requires_v3_after_ping_bootstrap_e2e`] |
|
||||
//!
|
||||
//! Two acceptance items are deliberately left to the in-process tests. A stale
|
||||
//! timestamp cannot be forged from outside — it is inside the HMAC — so
|
||||
@@ -91,20 +88,18 @@
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use crate::storage_api::internode_rpc_signature::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
|
||||
};
|
||||
use http::{HeaderMap, Method};
|
||||
use rustfs_config::{
|
||||
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
ENV_INTERNODE_RPC_SIGNATURE_STRICT,
|
||||
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_SIGNATURE_STRICT,
|
||||
};
|
||||
use rustfs_protos::canonical_make_volume_request_body;
|
||||
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
|
||||
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use tonic::{Code, Request, Response, Status};
|
||||
use tonic::{Code, Request, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
@@ -120,14 +115,12 @@ const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
|
||||
/// clears authentication stops harmlessly at `find_disk`.
|
||||
const ABSENT_DISK: &str = "/nonexistent/rustfs-signature-e2e-disk";
|
||||
|
||||
/// Wire names of the v2 and replay-scope headers these black-box tests edit. They are
|
||||
/// `pub(crate)` in ecstore, so they are repeated here rather than imported.
|
||||
/// [`overwrite_header`] asserts the header it replaces was actually present, which turns a
|
||||
/// rename into a loud failure instead of silently reducing an attack to a no-op.
|
||||
/// Wire names of the two v2 headers these tests edit. They are `pub(crate)` in
|
||||
/// ecstore, so they are repeated here rather than imported — [`overwrite_header`]
|
||||
/// asserts the header it replaces was actually present, which turns a rename
|
||||
/// into a loud failure instead of silently reducing an attack to a no-op.
|
||||
const CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||
const NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
const BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
|
||||
|
||||
/// gRPC service name carried in the signed scope, i.e. `TONIC_RPC_PREFIX`
|
||||
/// without its leading `/`.
|
||||
@@ -162,28 +155,19 @@ fn align_rpc_secret_with_server() {
|
||||
///
|
||||
/// Uses the no-cleanup spawn so a `pkill` pattern cannot reap servers belonging
|
||||
/// to other tests running in the same binary.
|
||||
fn server_env(extra_env: &[(&'static str, &'static str)]) -> Vec<(&'static str, &'static str)> {
|
||||
async fn start_server(extra_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut child_env = vec![
|
||||
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
|
||||
(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "false"),
|
||||
(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "false"),
|
||||
(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "false"),
|
||||
(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "1048576"),
|
||||
];
|
||||
child_env.extend_from_slice(extra_env);
|
||||
child_env
|
||||
}
|
||||
|
||||
async fn start_server_with_env(child_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(child_env).await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&child_env).await?;
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
async fn start_server(extra_env: &[(&'static str, &'static str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||
start_server_with_env(&server_env(extra_env)).await
|
||||
}
|
||||
|
||||
/// Stop the child and drop the cached gRPC channel for its address.
|
||||
///
|
||||
/// `node_service_time_out_client_no_auth` memoises channels in a process-global
|
||||
@@ -254,83 +238,12 @@ fn overwrite_header(headers: &mut HeaderMap, name: &'static str, value: &str) {
|
||||
/// and nothing else — no interceptor adds or rewrites auth metadata, so the
|
||||
/// bytes on the wire are the ones the test chose.
|
||||
async fn call_make_volume(url: &str, request: MakeVolumeRequest, headers: HeaderMap) -> Result<MakeVolumeResponse, Status> {
|
||||
call_make_volume_response(url, request, headers)
|
||||
.await
|
||||
.map(Response::into_inner)
|
||||
}
|
||||
|
||||
async fn call_make_volume_response(
|
||||
url: &str,
|
||||
request: MakeVolumeRequest,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Response<MakeVolumeResponse>, Status> {
|
||||
let mut client = node_service_time_out_client_no_auth(&url.to_string())
|
||||
.await
|
||||
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
|
||||
let mut rpc_request = Request::new(request);
|
||||
rpc_request.metadata_mut().as_mut().extend(headers);
|
||||
client.make_volume(rpc_request).await
|
||||
}
|
||||
|
||||
async fn call_ping_response(url: &str, headers: HeaderMap) -> Result<Response<PingResponse>, Status> {
|
||||
let mut client = node_service_time_out_client_no_auth(&url.to_string())
|
||||
.await
|
||||
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
|
||||
let mut rpc_request = Request::new(PingRequest {
|
||||
version: 1,
|
||||
body: bytes::Bytes::new(),
|
||||
});
|
||||
rpc_request.metadata_mut().as_mut().extend(headers);
|
||||
client.ping(rpc_request).await
|
||||
}
|
||||
|
||||
fn attach_boot_epoch_challenge(headers: &mut HeaderMap) -> Uuid {
|
||||
let challenge = Uuid::new_v4();
|
||||
headers.insert(
|
||||
BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
challenge.to_string().parse().expect("UUID must be a valid header value"),
|
||||
);
|
||||
challenge
|
||||
}
|
||||
|
||||
fn mint_replay_scope_headers(audience: &str, path: &str, content_sha256: &str, boot_epoch: Uuid) -> HeaderMap {
|
||||
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(content_sha256));
|
||||
let timestamp = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 headers must carry a timestamp")
|
||||
.to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers(audience, path, ×tamp, content_sha256, boot_epoch)
|
||||
.expect("replay-scope headers must mint with the aligned RPC secret"),
|
||||
);
|
||||
headers
|
||||
}
|
||||
|
||||
async fn learn_boot_epoch_from_make_volume(url: &str, audience: &str) -> Uuid {
|
||||
let request = make_volume_request("signature-e2e-epoch-bootstrap");
|
||||
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(&canonical_digest(&request)));
|
||||
let challenge = attach_boot_epoch_challenge(&mut headers);
|
||||
let response = call_make_volume_response(url, request, headers)
|
||||
.await
|
||||
.expect("v2 request with epoch challenge must clear default authentication");
|
||||
let boot_epoch = verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
|
||||
.expect("server must HMAC-authenticate the advertised boot epoch");
|
||||
assert_authenticated(
|
||||
Ok(response.into_inner()),
|
||||
"a v2 epoch-challenge request in the default replay-scope posture",
|
||||
);
|
||||
boot_epoch
|
||||
}
|
||||
|
||||
async fn learn_boot_epoch_from_ping(url: &str, audience: &str) -> Uuid {
|
||||
let mut headers = mint_v2_headers(audience, "Ping", None);
|
||||
let challenge = attach_boot_epoch_challenge(&mut headers);
|
||||
let response = call_ping_response(url, headers)
|
||||
.await
|
||||
.expect("v2 Ping with an epoch challenge must bootstrap strict replay scope");
|
||||
verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
|
||||
.expect("strict replay-scope Ping must return a valid boot epoch proof")
|
||||
client.make_volume(rpc_request).await.map(|response| response.into_inner())
|
||||
}
|
||||
|
||||
/// Assert a call cleared authentication.
|
||||
@@ -419,122 +332,6 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A replay-scoped signature is usable exactly once against the exact gRPC path and the server
|
||||
/// process epoch that minted it. This crosses the child-process boundary twice: the HMAC-protected
|
||||
/// epoch is learned from a real response, then the same server is restarted in place to prove its
|
||||
/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
|
||||
init_logging();
|
||||
align_rpc_secret_with_server();
|
||||
let child_env = server_env(&[]);
|
||||
let mut env = start_server_with_env(&child_env).await?;
|
||||
let url = env.url.clone();
|
||||
let audience = audience_of(&env);
|
||||
let boot_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
|
||||
|
||||
let request = make_volume_request("replay-scope-e2e-once");
|
||||
let captured = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
boot_epoch,
|
||||
);
|
||||
assert_authenticated(
|
||||
call_make_volume(&url, request.clone(), captured.clone()).await,
|
||||
"the first replay-scoped mutation delivery",
|
||||
);
|
||||
assert_rejected(
|
||||
call_make_volume(&url, request.clone(), captured).await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"the same replay-scoped mutation delivered twice",
|
||||
);
|
||||
|
||||
let transplanted =
|
||||
mint_replay_scope_headers(&audience, &format!("{TONIC_RPC_PREFIX}/Ping"), &canonical_digest(&request), boot_epoch);
|
||||
assert_rejected(
|
||||
call_make_volume(&url, request.clone(), transplanted).await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"a replay-scoped Ping signature transplanted onto MakeVolume",
|
||||
);
|
||||
|
||||
let stale_epoch = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
boot_epoch,
|
||||
);
|
||||
env.restart_server_preserving_data(Vec::new(), &child_env).await?;
|
||||
rustfs_protos::evict_failed_connection(&url).await;
|
||||
assert_rejected(
|
||||
call_make_volume(&url, request.clone(), stale_epoch).await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"a replay-scoped signature captured before the receiving process restart",
|
||||
);
|
||||
|
||||
let restarted_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
|
||||
assert_ne!(boot_epoch, restarted_epoch, "a restarted child process must advertise a new boot epoch");
|
||||
let fresh_epoch = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
restarted_epoch,
|
||||
);
|
||||
assert_authenticated(
|
||||
call_make_volume(&url, request, fresh_epoch).await,
|
||||
"a replay-scoped mutation signed with the replacement process epoch",
|
||||
);
|
||||
|
||||
stop_server(env, &url).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Strict replay scope leaves one authenticated v2 bootstrap: `Ping` carrying a fresh challenge.
|
||||
/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
|
||||
/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
|
||||
init_logging();
|
||||
align_rpc_secret_with_server();
|
||||
let env = start_server(&[(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "true")]).await?;
|
||||
let url = env.url.clone();
|
||||
let audience = audience_of(&env);
|
||||
|
||||
let v2_request = make_volume_request("replay-scope-e2e-strict-v2");
|
||||
assert_rejected(
|
||||
call_make_volume(
|
||||
&url,
|
||||
v2_request.clone(),
|
||||
mint_v2_headers(&audience, "MakeVolume", Some(&canonical_digest(&v2_request))),
|
||||
)
|
||||
.await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"a v2 mutation after replay-scope strictness is enabled",
|
||||
);
|
||||
|
||||
let boot_epoch = learn_boot_epoch_from_ping(&url, &audience).await;
|
||||
let request = make_volume_request("replay-scope-e2e-strict-v3");
|
||||
let replay_scoped = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
boot_epoch,
|
||||
);
|
||||
assert_authenticated(
|
||||
call_make_volume(&url, request, replay_scoped).await,
|
||||
"a replay-scoped mutation after Ping bootstrap under strict replay scope",
|
||||
);
|
||||
|
||||
stop_server(env, &url).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Baseline: correctly signed mutations are accepted, both with and without a
|
||||
/// body digest.
|
||||
///
|
||||
|
||||
@@ -30,7 +30,6 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
@@ -69,9 +68,7 @@ pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
|
||||
}
|
||||
|
||||
pub fn sse_customer_key_md5_base64(key: &str) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(key.as_bytes());
|
||||
BASE64.encode(hasher.finalize())
|
||||
BASE64.encode(md5::compute(key).0)
|
||||
}
|
||||
|
||||
pub async fn kms_admin_request(
|
||||
|
||||
@@ -25,18 +25,12 @@ use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use md5::compute;
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{info, warn};
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Test encryption of zero-byte files (empty files)
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -300,7 +294,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
|
||||
info!("🔍 Testing invalid SSE-C key length");
|
||||
let invalid_short_key = "short"; // Too short
|
||||
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
|
||||
let invalid_key_md5 = md5_hex(invalid_short_key);
|
||||
let invalid_key_md5 = format!("{:x}", compute(invalid_short_key));
|
||||
|
||||
let invalid_key_result = s3_client
|
||||
.put_object()
|
||||
|
||||
@@ -26,7 +26,6 @@ use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
@@ -51,15 +50,7 @@ fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
}
|
||||
|
||||
fn sse_customer_key_md5_base64(key: &str) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(key.as_bytes());
|
||||
base64::engine::general_purpose::STANDARD.encode(hasher.finalize())
|
||||
}
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
base64::engine::general_purpose::STANDARD.encode(md5::compute(key).0)
|
||||
}
|
||||
|
||||
/// Env var consumed by the local SSE-S3 DEK provider when KMS is not configured.
|
||||
@@ -5673,7 +5664,7 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let archive = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await;
|
||||
let expected_etag = format!("\"{}\"", md5_hex(&archive));
|
||||
let expected_etag = format!("\"{:x}\"", md5::compute(&archive));
|
||||
|
||||
let response = client
|
||||
.put_object()
|
||||
|
||||
@@ -21,22 +21,18 @@
|
||||
//! function, never as an S3 event sink.
|
||||
//!
|
||||
//! Coverage:
|
||||
//! * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct
|
||||
//! * PUT / multipart-complete / DELETE each deliver one event with the correct
|
||||
//! eventName, bucket, key, versionId and eTag.
|
||||
//! * prefix/suffix filters drop non-matching keys (rule-engine gate).
|
||||
//! * an event queued while the target endpoint is unreachable is redelivered
|
||||
//! from the on-disk store once the endpoint recovers (store-and-forward).
|
||||
//! * responseElements and the S3 response use the canonical request ID while
|
||||
//! requestParameters preserve a conflicting client-supplied value.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::operation::RequestId;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, Event, FilterRule, FilterRuleName,
|
||||
NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
|
||||
VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Event, FilterRule, FilterRuleName,
|
||||
NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter, VersioningConfiguration,
|
||||
};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use local_ip_address::local_ip;
|
||||
@@ -44,12 +40,10 @@ use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc, Once,
|
||||
@@ -69,8 +63,6 @@ type BoxError = Box<dyn Error + Send + Sync>;
|
||||
/// for target lookup (see `process_queue_configurations`), so the region is
|
||||
/// nominal, but it must be present for `ARN::parse` to succeed.
|
||||
const NOTIFY_REGION: &str = "us-east-1";
|
||||
const CLIENT_REQUEST_ID: &str = "client-supplied-request-id";
|
||||
const CLIENT_AMZ_REQUEST_ID: &str = "client-supplied-amz-request-id";
|
||||
|
||||
/// Webhook targets are registered as `TargetID { id: <name>, name: "webhook" }`,
|
||||
/// so the ARN a notification rule references is
|
||||
@@ -587,36 +579,6 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
|
||||
value.map(|e| e.trim_matches('"').to_string())
|
||||
}
|
||||
|
||||
fn assert_conflicting_request_id_correlation(record: &Value, server_request_id: &str) {
|
||||
assert_eq!(
|
||||
record["requestParameters"][REQUEST_ID_HEADER].as_str(),
|
||||
Some(CLIENT_REQUEST_ID),
|
||||
"notification request parameters should retain the actual client header: {record}"
|
||||
);
|
||||
assert_eq!(
|
||||
record["requestParameters"][AMZ_REQUEST_ID].as_str(),
|
||||
Some(CLIENT_AMZ_REQUEST_ID),
|
||||
"notification request parameters should retain the actual client header: {record}"
|
||||
);
|
||||
assert_eq!(
|
||||
record["responseElements"][AMZ_REQUEST_ID].as_str(),
|
||||
Some(server_request_id),
|
||||
"notification response elements should use the canonical request ID: {record}"
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_generated_request_id_correlation(record: &Value, request_id: &str) {
|
||||
assert!(
|
||||
record["requestParameters"][AMZ_REQUEST_ID].is_null(),
|
||||
"notification request parameters must not invent a client request header: {record}"
|
||||
);
|
||||
assert_eq!(
|
||||
record["responseElements"][AMZ_REQUEST_ID].as_str(),
|
||||
Some(request_id),
|
||||
"notification response elements should match the S3 response request ID: {record}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -709,17 +671,8 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
.bucket(bucket)
|
||||
.key(put_key)
|
||||
.body(ByteStream::from_static(b"peri-1 put body"))
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert(REQUEST_ID_HEADER, CLIENT_REQUEST_ID);
|
||||
request.headers_mut().insert(AMZ_REQUEST_ID, CLIENT_AMZ_REQUEST_ID);
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let put_request_id = put.request_id().ok_or("PUT response missing request ID")?.to_owned();
|
||||
assert!(uuid::Uuid::parse_str(&put_request_id).is_ok());
|
||||
assert_ne!(put_request_id, CLIENT_REQUEST_ID);
|
||||
assert_ne!(put_request_id, CLIENT_AMZ_REQUEST_ID);
|
||||
let put_version = put
|
||||
.version_id()
|
||||
.ok_or("PUT response missing versionId (versioning not enabled?)")?;
|
||||
@@ -739,7 +692,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
"record eventName: {record}"
|
||||
);
|
||||
assert_eq!(record["s3"]["bucket"]["name"].as_str(), Some(bucket), "bucket in event: {record}");
|
||||
assert_conflicting_request_id_correlation(record, &put_request_id);
|
||||
assert_eq!(object["versionId"].as_str(), Some(put_version), "versionId in event: {object}");
|
||||
assert_eq!(
|
||||
trimmed_etag(object["eTag"].as_str()),
|
||||
@@ -792,35 +744,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
"multipart eTag in event: {mp_record}"
|
||||
);
|
||||
|
||||
// --- Snowball extract: direct notification path keeps response correlation
|
||||
let snowball_key = "uploads/snowball.dat";
|
||||
let snowball_body = b"snowball notification body";
|
||||
let mut archive_builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
let mut archive_header = tokio_tar::Header::new_gnu();
|
||||
archive_header.set_size(u64::try_from(snowball_body.len()).expect("snowball fixture length should fit in u64"));
|
||||
archive_header.set_mode(0o644);
|
||||
archive_header.set_cksum();
|
||||
archive_builder
|
||||
.append_data(&mut archive_header, snowball_key, Cursor::new(snowball_body))
|
||||
.await?;
|
||||
let archive = archive_builder.into_inner().await?.into_inner();
|
||||
let snowball = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("snowball-fixture.tar")
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let snowball_request_id = snowball.request_id().ok_or("Snowball response missing request ID")?;
|
||||
|
||||
let snowball_event = wait_for_event(&mut rx, snowball_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
|
||||
let snowball_record = &snowball_event["Records"][0];
|
||||
assert_generated_request_id_correlation(snowball_record, snowball_request_id);
|
||||
|
||||
// --- Filter: non-matching prefix and suffix must never be delivered ------
|
||||
let wrong_prefix = "logs/report.dat"; // right suffix, wrong prefix
|
||||
let wrong_suffix = "uploads/report.txt"; // right prefix, wrong suffix
|
||||
@@ -849,32 +772,7 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
"wrong-suffix key {wrong_suffix} bypassed the filter; saw {seen:?}"
|
||||
);
|
||||
|
||||
// --- DeleteObjects: direct notification path keeps response correlation --
|
||||
let delete_many_key = "uploads/delete-many.dat";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(delete_many_key)
|
||||
.body(ByteStream::from_static(b"delete objects notification body"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_event(&mut rx, delete_many_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
|
||||
|
||||
let delete_many = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key(delete_many_key).build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let delete_many_request_id = delete_many.request_id().ok_or("DeleteObjects response missing request ID")?;
|
||||
let delete_many_event = wait_for_event(&mut rx, delete_many_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
|
||||
assert_generated_request_id_correlation(&delete_many_event["Records"][0], delete_many_request_id);
|
||||
|
||||
// --- DeleteObject on a versioned bucket: delete-marker version ----------
|
||||
// --- DELETE on a versioned bucket: ObjectRemoved:* with delete-marker version
|
||||
let delete = client.delete_object().bucket(bucket).key(put_key).send().await?;
|
||||
let removed = wait_for_event(&mut rx, put_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
|
||||
let removed_record = &removed["Records"][0];
|
||||
|
||||
@@ -24,7 +24,7 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
|
||||
use tonic::Request;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::storage_api::grpc_lock::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
use crate::storage_api::grpc_lock::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
|
||||
/// gRPC lock client without authentication for testing
|
||||
/// Similar to RemoteClient but uses no_auth client
|
||||
@@ -42,7 +42,7 @@ impl GrpcLockClient {
|
||||
&self,
|
||||
) -> Result<
|
||||
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
|
||||
tonic::service::interceptor::InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, TonicInterceptor>,
|
||||
>,
|
||||
> {
|
||||
node_service_time_out_client_no_auth(&self.addr)
|
||||
|
||||
@@ -23,8 +23,7 @@ use rustfs_protos::{
|
||||
proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
|
||||
SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, node_service_server::NodeService,
|
||||
node_service_server::NodeService,
|
||||
},
|
||||
};
|
||||
use std::pin::Pin;
|
||||
@@ -105,27 +104,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseRenewRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseReleaseRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
|
||||
let request = request.into_inner();
|
||||
let args: LockRequest = match serde_json::from_str(&request.args) {
|
||||
|
||||
@@ -95,7 +95,6 @@ const MANUAL_ASYNC_PARALLEL_OBJECTS: usize = 64;
|
||||
const MANUAL_ACTIVE_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_RESTART_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT: StdDuration = StdDuration::from_secs(15);
|
||||
const MANUAL_TRANSITION_CANCEL_BARRIER_ENV: &str = "RUSTFS_E2E_MANUAL_TRANSITION_CANCEL_BARRIER";
|
||||
const MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT: StdDuration = StdDuration::from_secs(90);
|
||||
const MANUAL_RESTART_RECOVERY_TIMEOUT: StdDuration = StdDuration::from_secs(80);
|
||||
const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
|
||||
@@ -1685,15 +1684,8 @@ async fn test_manual_transition_async_active_cancel_reports_terminal_cancelled()
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
@@ -103,12 +102,10 @@ impl Intercept for ResponseHeaderCapture {
|
||||
|
||||
fn customer_key(byte: u8) -> CustomerKey {
|
||||
let raw = [byte; 32];
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(raw);
|
||||
CustomerKey {
|
||||
raw: String::from_utf8_lossy(&raw).into_owned(),
|
||||
encoded: base64::engine::general_purpose::STANDARD.encode(raw),
|
||||
md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()),
|
||||
md5: base64::engine::general_purpose::STANDARD.encode(md5::compute(raw).0),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,12 +16,9 @@
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
verify_tonic_boot_epoch_response,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
|
||||
@@ -33,7 +30,7 @@ pub(crate) mod node_interact {
|
||||
}
|
||||
|
||||
pub(crate) mod grpc_lock {
|
||||
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
pub(crate) use super::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
}
|
||||
|
||||
/// Signing/transport surface used by the cross-process internode RPC signature
|
||||
@@ -43,8 +40,7 @@ pub(crate) mod grpc_lock {
|
||||
#[cfg(test)]
|
||||
pub(crate) mod internode_rpc_signature {
|
||||
pub(crate) use super::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -153,18 +153,11 @@ metrics = { workspace = true }
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rustfs-uring = "0.2.1"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi-util.workspace = true
|
||||
windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
# Only for `pin_callsite_interest_for_test`, which registers a `NoSubscriber`
|
||||
# dispatcher to keep tracing's process-global callsite-interest cache honest.
|
||||
tracing-core = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
proptest = "1"
|
||||
|
||||
@@ -67,14 +67,6 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod transition_transaction {
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod evaluator {
|
||||
pub use crate::bucket::lifecycle::evaluator::Evaluator;
|
||||
}
|
||||
@@ -130,13 +122,13 @@ pub mod bucket {
|
||||
|
||||
pub mod metadata_sys {
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataSys, acquire_bucket_metadata_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
|
||||
BucketMetadataSys, acquire_bucket_targets_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with, update_under_transaction_lock,
|
||||
init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
update_bucket_targets_under_transaction_lock, update_config_with,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -184,8 +176,8 @@ pub mod bucket {
|
||||
get_global_replication_stats, init_background_replication, read_durable_mrf_backlog, replication_state_to_filemeta,
|
||||
replication_status_to_filemeta, replication_statuses_map, replication_target_arns, resync_start_conflict_id,
|
||||
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
should_use_existing_delete_replication_source, validate_replication_config_target_arns,
|
||||
version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -313,8 +305,7 @@ pub mod disk {
|
||||
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
|
||||
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
@@ -385,7 +376,6 @@ pub mod metrics {
|
||||
pub mod notification {
|
||||
pub use crate::services::notification_sys::{
|
||||
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -416,15 +406,13 @@ pub mod rio {
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
|
||||
PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
|
||||
ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
|
||||
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -554,7 +554,6 @@ async fn delete_free_version_remote_object(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let version_id_exact = validate_transition_remote_version(oi)?;
|
||||
let identity = tier_destination_id_from_metadata(&oi.user_defined)?
|
||||
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
@@ -563,7 +562,7 @@ async fn delete_free_version_remote_object(
|
||||
&oi.transitioned_object.tier,
|
||||
identity,
|
||||
tier_config_mgr,
|
||||
version_id_exact,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -4202,23 +4201,6 @@ pub async fn get_transitioned_object_reader(
|
||||
get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr).await
|
||||
}
|
||||
|
||||
fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::Error> {
|
||||
let version = oi.transitioned_object.version_id.as_str();
|
||||
match oi.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state is unknown",
|
||||
)),
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled if version.is_empty() => Ok(false),
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull if version == "null" => Ok(true),
|
||||
rustfs_filemeta::TransitionVersionState::Exact if !version.is_empty() && version != "null" => Ok(true),
|
||||
_ => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state conflicts with its version ID",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -4228,7 +4210,6 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
opts: &ObjectOptions,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<GetObjectReader, std::io::Error> {
|
||||
validate_transition_remote_version(oi)?;
|
||||
let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?;
|
||||
let lease = match expected_identity {
|
||||
Some(identity) => {
|
||||
@@ -5525,7 +5506,6 @@ mod tests {
|
||||
tier: tier.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -5589,7 +5569,6 @@ mod tests {
|
||||
tier,
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -5612,70 +5591,6 @@ mod tests {
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 1,
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: String::new(),
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier,
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("unknown remote version state must fail before backend IO"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let backend = register_mock_tier(&manager, "WARM").await;
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect_err("unknown remote version state must fail before backend IO");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_remote_delete_requires_persisted_destination_identity() {
|
||||
@@ -5725,7 +5640,6 @@ mod tests {
|
||||
oi.transitioned_object.tier = "WARM".to_string();
|
||||
oi.transitioned_object.name = "remote/object".to_string();
|
||||
oi.transitioned_object.version_id = "remote-version".to_string();
|
||||
oi.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
let local_delete_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
|
||||
let legacy_err = delete_free_version_remote_object_then(&oi, &manager, {
|
||||
@@ -5736,8 +5650,7 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect_err("legacy free-version without identity must be retained");
|
||||
assert_eq!(legacy_err.kind(), std::io::ErrorKind::Other);
|
||||
assert_eq!(old_backend.remove_count().await, 0);
|
||||
assert!(legacy_err.to_string().contains("no durable backend identity"));
|
||||
assert_eq!(local_delete_calls.load(Ordering::Relaxed), 0);
|
||||
|
||||
let mut invalid_metadata = HashMap::new();
|
||||
@@ -5868,7 +5781,6 @@ mod tests {
|
||||
oi.transitioned_object.tier = "WARM".to_string();
|
||||
oi.transitioned_object.name = "remote/object".to_string();
|
||||
oi.transitioned_object.version_id = "remote-version".to_string();
|
||||
oi.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
"bucket",
|
||||
@@ -5884,12 +5796,7 @@ mod tests {
|
||||
Ok(_) => panic!("identity-bound GET must reject a same-name tier rebind"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Other);
|
||||
let admin_err = err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<crate::client::admin_handler_utils::AdminError>())
|
||||
.expect("identity mismatch should retain the typed tier error");
|
||||
assert_eq!(admin_err.code, crate::services::tier::tier::ERR_TIER_INVALID_CONFIG.code);
|
||||
assert!(err.to_string().contains("identity no longer matches"));
|
||||
assert_eq!(new_backend.get_count().await, 0);
|
||||
|
||||
oi.user_defined = Arc::new(HashMap::new());
|
||||
@@ -5995,8 +5902,7 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([1; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
|
||||
let err = state
|
||||
@@ -6107,8 +6013,7 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([1; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
|
||||
state
|
||||
@@ -8974,7 +8879,7 @@ mod tests {
|
||||
.await
|
||||
.expect("first worker result should persist");
|
||||
assert_eq!(first.state, ManualTransitionJobState::Running);
|
||||
assert_eq!(first.report.transition_completed, 0);
|
||||
assert_eq!(first.report.transition_completed, 1);
|
||||
assert_eq!(first.report.transition_failed, 0);
|
||||
|
||||
let duplicate = record_manual_transition_worker_result(
|
||||
@@ -8987,11 +8892,11 @@ mod tests {
|
||||
.await
|
||||
.expect("duplicate worker result should be idempotent");
|
||||
assert_eq!(duplicate.state, ManualTransitionJobState::Running);
|
||||
assert_eq!(duplicate.report.transition_completed, 0);
|
||||
assert_eq!(duplicate.report.transition_completed, 1);
|
||||
assert_eq!(duplicate.report.transition_failed, 0);
|
||||
|
||||
let second_key = manual_transition_worker_result_task_key(&bucket, "logs/b", None);
|
||||
let pending_record = record_manual_transition_worker_result(
|
||||
let final_record = record_manual_transition_worker_result(
|
||||
ecstore.clone(),
|
||||
job_id,
|
||||
&second_key,
|
||||
@@ -9000,14 +8905,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("second distinct worker result should persist");
|
||||
assert_eq!(pending_record.state, ManualTransitionJobState::Running);
|
||||
assert_eq!(pending_record.report.transition_completed, 0);
|
||||
assert_eq!(pending_record.report.transition_failed, 0);
|
||||
|
||||
let final_record =
|
||||
reconcile_manual_transition_worker_results(ecstore.clone(), job_id, ManualTransitionQueueSnapshot::default())
|
||||
.await
|
||||
.expect("worker result journal should reconcile");
|
||||
assert_eq!(final_record.state, ManualTransitionJobState::Partial);
|
||||
assert_eq!(final_record.report.transition_completed, 1);
|
||||
assert_eq!(final_record.report.transition_failed, 1);
|
||||
@@ -9042,7 +8940,7 @@ mod tests {
|
||||
.expect("worker result job record should save");
|
||||
|
||||
let task_key = manual_transition_worker_result_task_key(&bucket, "logs/fail", None);
|
||||
let pending_record = record_manual_transition_worker_result_with_reason(
|
||||
let final_record = record_manual_transition_worker_result_with_reason(
|
||||
ecstore.clone(),
|
||||
job_id,
|
||||
&task_key,
|
||||
@@ -9052,12 +8950,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("worker result with failure reason should persist");
|
||||
assert!(pending_record.report.tier_failure_by_reason.is_empty());
|
||||
assert_eq!(pending_record.report.transition_failed, 0);
|
||||
|
||||
let final_record = reconcile_manual_transition_worker_results(ecstore, job_id, ManualTransitionQueueSnapshot::default())
|
||||
.await
|
||||
.expect("worker failure reason should reconcile");
|
||||
assert_eq!(
|
||||
final_record
|
||||
.report
|
||||
@@ -10188,87 +10081,6 @@ mod tests {
|
||||
(backend, identity_hex)
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
|
||||
let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available")
|
||||
.backend_identity();
|
||||
let je = Jentry {
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: false,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
};
|
||||
|
||||
let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
.await
|
||||
.expect_err("unknown journal state must fail before backend IO");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_replay_deletes_confirmed_exact_provider_token() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
backend
|
||||
.set_put_remote_version(Some("provider-version-token".to_string()))
|
||||
.await;
|
||||
lease
|
||||
.put(
|
||||
"remote/object",
|
||||
crate::client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")),
|
||||
9,
|
||||
)
|
||||
.await
|
||||
.expect("confirmed remote candidate should be seeded");
|
||||
backend.set_remove_failure(true);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
let je = Jentry {
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "provider-version-token".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
};
|
||||
|
||||
crate::set_disk::cleanup_rejected_transition_upload_durably(
|
||||
&lease,
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
true,
|
||||
Some(ecstore.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("failed immediate cleanup should remain durable in the journal");
|
||||
assert!(backend.contains(&je.obj_name).await);
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
.await
|
||||
.expect("identity-bound exact journal must retry confirmed candidate cleanup");
|
||||
|
||||
assert!(!backend.contains(&je.obj_name).await);
|
||||
assert_eq!(backend.exact_remove_count(), 2);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), "provider-version-token".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed_recoverable_free_version(
|
||||
disk_paths: &[PathBuf],
|
||||
bucket: &str,
|
||||
@@ -10288,7 +10100,6 @@ mod tests {
|
||||
identity,
|
||||
);
|
||||
}
|
||||
let transition_version_id = Uuid::new_v4();
|
||||
let mut metadata = FileMeta::new();
|
||||
metadata
|
||||
.add_version(FileInfo {
|
||||
@@ -10297,9 +10108,7 @@ mod tests {
|
||||
version_id: Some(object_version_id),
|
||||
transition_status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
transitioned_objname: format!("remote/{bucket}/{object}"),
|
||||
transition_version_id: Some(transition_version_id),
|
||||
transition_version: Some(transition_version_id.to_string()),
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
transition_version_id: Some(Uuid::new_v4()),
|
||||
transition_tier: "WARM".to_string(),
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
metadata: transitioned_metadata,
|
||||
@@ -11318,25 +11127,23 @@ mod tests {
|
||||
|
||||
// Distinct payloads with distinct sizes: a mixed-generation reassembly
|
||||
// would produce bytes matching none of them (or fail the read outright).
|
||||
let candidates: Vec<Vec<u8>> = (0..2)
|
||||
let candidates: Vec<Vec<u8>> = (0..3)
|
||||
.map(|g| {
|
||||
let len = 4096 + g * 512;
|
||||
vec![b'a' + g as u8; len]
|
||||
})
|
||||
.collect();
|
||||
|
||||
let commit_barrier = MultipartCommitBarrier::install_for_arrivals(
|
||||
&bucket,
|
||||
object,
|
||||
MultipartCommitPause::PutPartBeforeLockAcquire,
|
||||
candidates.len(),
|
||||
);
|
||||
let commit_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
let start = Arc::new(tokio::sync::Barrier::new(candidates.len() + 1));
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
for payload in candidates.iter().cloned() {
|
||||
let store = ecstore.clone();
|
||||
let bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
let start = Arc::clone(&start);
|
||||
tasks.spawn(async move {
|
||||
start.wait().await;
|
||||
let mut data = PutObjReader::from_vec(payload.clone());
|
||||
store
|
||||
.put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
|
||||
@@ -11344,10 +11151,11 @@ mod tests {
|
||||
.map(|info| (info, payload))
|
||||
});
|
||||
}
|
||||
start.wait().await;
|
||||
|
||||
// Both writers finish streaming before racing for the uploadId commit
|
||||
// lock. Two generations are sufficient to exercise the mixed-shard
|
||||
// hazard, while each waiter sits behind at most one cross-disk rename.
|
||||
// The first writer holds the uploadId commit lock while the other
|
||||
// resends reach the same critical section. Releasing it proves the
|
||||
// handoff without depending on saturated CI disk latency.
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release();
|
||||
|
||||
|
||||
@@ -1646,14 +1646,40 @@ pub async fn record_manual_transition_worker_result_with_reason(
|
||||
job_id: Uuid,
|
||||
task_key: &str,
|
||||
result: ManualTransitionWorkerResult,
|
||||
_queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let result_record = ManualTransitionWorkerResultRecord::new_with_reason(job_id, task_key, result, failure_reason);
|
||||
if !save_manual_transition_worker_result_if_absent(api.clone(), &result_record).await? {
|
||||
return load_manual_transition_job_record(api, job_id).await;
|
||||
}
|
||||
load_manual_transition_job_record(api, job_id).await
|
||||
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if record.is_terminal() {
|
||||
return Ok(record);
|
||||
}
|
||||
record.record_worker_result_with_reason(result, queue_snapshot, failure_reason);
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
if record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(
|
||||
api.clone(),
|
||||
&record.scope_key,
|
||||
record.job_id,
|
||||
record.lease_id,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
|
||||
pub async fn renew_manual_transition_job_lease(
|
||||
|
||||
@@ -20,10 +20,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
@@ -45,7 +42,6 @@ const TIER_DELETE_JOURNAL_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
|
||||
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
|
||||
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -59,35 +55,24 @@ struct PersistedTierDeleteJournalEntry {
|
||||
backend_identity: Option<[u8; 32]>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
version_id_exact: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
version_state: Option<rustfs_filemeta::TransitionVersionState>,
|
||||
}
|
||||
|
||||
impl PersistedTierDeleteJournalEntry {
|
||||
fn from_jentry(je: &Jentry) -> Result<Self> {
|
||||
validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?;
|
||||
let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
let version = if legacy_unknown {
|
||||
if je.backend_identity.is_some() {
|
||||
fn from_jentry(je: &Jentry) -> Self {
|
||||
Self {
|
||||
version: if je.version_id_exact {
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION
|
||||
} else if je.backend_identity.is_some() {
|
||||
TIER_DELETE_JOURNAL_VERSION
|
||||
} else {
|
||||
1
|
||||
}
|
||||
} else {
|
||||
if je.backend_identity.is_none() {
|
||||
return Err(Error::other("new tier delete journal entry is missing its backend identity"));
|
||||
}
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION
|
||||
};
|
||||
Ok(Self {
|
||||
version,
|
||||
},
|
||||
obj_name: je.obj_name.clone(),
|
||||
version_id: je.version_id.clone(),
|
||||
tier_name: je.tier_name.clone(),
|
||||
backend_identity: je.backend_identity,
|
||||
version_id_exact: je.version_id_exact.then_some(true),
|
||||
version_state: (!legacy_unknown).then_some(je.version_state),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn into_jentry(self) -> Result<Jentry> {
|
||||
@@ -99,23 +84,19 @@ impl PersistedTierDeleteJournalEntry {
|
||||
if self.obj_name.is_empty() || self.tier_name.is_empty() {
|
||||
return Err(Error::other("tier delete journal entry is incomplete"));
|
||||
}
|
||||
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION
|
||||
&& self.version != TIER_DELETE_JOURNAL_STATE_VERSION
|
||||
&& self.version_id_exact.unwrap_or(false)
|
||||
{
|
||||
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION && self.version_id_exact.unwrap_or(false) {
|
||||
return Err(Error::other(
|
||||
"legacy tier delete journal entry has an unsupported exact version constraint",
|
||||
));
|
||||
}
|
||||
let (backend_identity, version_id_exact, version_state) = match self.version {
|
||||
1 => (None, false, rustfs_filemeta::TransitionVersionState::Unknown),
|
||||
let (backend_identity, version_id_exact) = match self.version {
|
||||
1 => (None, false),
|
||||
TIER_DELETE_JOURNAL_VERSION => (
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v2 entry is missing its backend identity"))?,
|
||||
),
|
||||
false,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
),
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION => {
|
||||
if self.version_id.is_empty() || self.version_id_exact != Some(true) {
|
||||
@@ -127,22 +108,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
.ok_or_else(|| Error::other("tier delete journal v3 entry is missing its backend identity"))?,
|
||||
),
|
||||
true,
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
)
|
||||
}
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION => {
|
||||
let state = self
|
||||
.version_state
|
||||
.ok_or_else(|| Error::other("tier delete journal v4 entry is missing its version state"))?;
|
||||
let exact = self.version_id_exact.unwrap_or(false);
|
||||
validate_version_state(state, &self.version_id, exact)?;
|
||||
(
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v4 entry is missing its backend identity"))?,
|
||||
),
|
||||
exact,
|
||||
state,
|
||||
)
|
||||
}
|
||||
version => return Err(Error::other(format!("unsupported tier delete journal version {version}"))),
|
||||
@@ -153,30 +118,10 @@ impl PersistedTierDeleteJournalEntry {
|
||||
tier_name: self.tier_name,
|
||||
backend_identity,
|
||||
version_id_exact,
|
||||
version_state,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_version_state(
|
||||
state: rustfs_filemeta::TransitionVersionState,
|
||||
version_id: &str,
|
||||
version_id_exact: bool,
|
||||
) -> Result<()> {
|
||||
use rustfs_filemeta::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull, Unknown};
|
||||
|
||||
let valid = match state {
|
||||
Unknown => !version_id_exact,
|
||||
KnownDisabled => version_id.is_empty() && !version_id_exact,
|
||||
SuspendedNull => version_id == "null" && version_id_exact,
|
||||
Exact => !version_id.is_empty() && version_id != "null" && version_id_exact,
|
||||
};
|
||||
if !valid {
|
||||
return Err(Error::other("tier delete journal version state conflicts with its version id"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TierDeleteJournalRecoveryStats {
|
||||
pub scanned: usize,
|
||||
@@ -214,7 +159,7 @@ pub(crate) fn decode_tier_delete_journal_entry(data: &[u8]) -> Result<Jentry> {
|
||||
}
|
||||
|
||||
pub(crate) fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
|
||||
serde_json::to_vec(&PersistedTierDeleteJournalEntry::from_jentry(je)?)
|
||||
serde_json::to_vec(&PersistedTierDeleteJournalEntry::from_jentry(je))
|
||||
.map_err(|err| Error::other(format!("encode tier delete journal failed: {err}")))
|
||||
}
|
||||
|
||||
@@ -264,35 +209,18 @@ where
|
||||
}
|
||||
|
||||
pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
if je.version_state == rustfs_filemeta::TransitionVersionState::Unknown {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"tier delete journal remote version state is unknown",
|
||||
));
|
||||
}
|
||||
let backend_identity = je
|
||||
.backend_identity
|
||||
.ok_or_else(|| std::io::Error::other("legacy tier delete journal has no durable backend identity"))?;
|
||||
if je.version_id_exact {
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
je.version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
@@ -478,9 +406,8 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, await_tier_delete_journal_recovery,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
tier_delete_journal_object_name,
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, await_tier_delete_journal_recovery, decode_tier_delete_journal_entry,
|
||||
encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity, tier_delete_journal_object_name,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::error::Result;
|
||||
@@ -493,8 +420,7 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([7; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -510,7 +436,6 @@ mod tests {
|
||||
assert_eq!(decoded.tier_name, je.tier_name);
|
||||
assert_eq!(decoded.backend_identity, je.backend_identity);
|
||||
assert_eq!(decoded.version_id_exact, je.version_id_exact);
|
||||
assert_eq!(decoded.version_state, je.version_state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -525,7 +450,7 @@ mod tests {
|
||||
let persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("exact journal JSON should decode");
|
||||
let decoded = decode_tier_delete_journal_entry(&encoded).expect("exact journal entry should decode");
|
||||
|
||||
assert_eq!(persisted["version"], TIER_DELETE_JOURNAL_STATE_VERSION);
|
||||
assert_eq!(persisted["version"], TIER_DELETE_JOURNAL_EXACT_VERSION);
|
||||
assert_eq!(persisted["version_id_exact"], true);
|
||||
assert!(decoded.version_id_exact);
|
||||
assert_ne!(tier_delete_journal_object_name(&exact), tier_delete_journal_object_name(&normalized));
|
||||
@@ -588,46 +513,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_rejects_conflicting_v4_version_states() {
|
||||
let identity = vec![7_u8; 32];
|
||||
let invalid = [
|
||||
("known-disabled", "unexpected", false),
|
||||
("suspended-null", "", true),
|
||||
("suspended-null", "null", false),
|
||||
("exact", "", true),
|
||||
("exact", "null", true),
|
||||
("exact", "version", false),
|
||||
("unknown", "version", true),
|
||||
];
|
||||
|
||||
for (state, version_id, exact) in invalid {
|
||||
let persisted = serde_json::json!({
|
||||
"version": TIER_DELETE_JOURNAL_STATE_VERSION,
|
||||
"obj_name": "remote/object",
|
||||
"version_id": version_id,
|
||||
"tier_name": "WARM",
|
||||
"backend_identity": identity,
|
||||
"version_id_exact": exact.then_some(true),
|
||||
"version_state": state,
|
||||
});
|
||||
let encoded = serde_json::to_vec(&persisted).expect("invalid journal fixture should encode");
|
||||
decode_tier_delete_journal_entry(&encoded).expect_err("conflicting v4 version state must fail closed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_journals_decode_with_unknown_version_state() {
|
||||
let v1 = br#"{"version":1,"obj_name":"remote/object","version_id":"opaque","tier_name":"WARM"}"#;
|
||||
let v2 = br#"{"version":2,"obj_name":"remote/object","version_id":"opaque","tier_name":"WARM","backend_identity":[7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]}"#;
|
||||
|
||||
for payload in [v1.as_slice(), v2.as_slice()] {
|
||||
let decoded = decode_tier_delete_journal_entry(payload).expect("legacy journal should decode");
|
||||
assert_eq!(decoded.version_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
assert!(!decoded.version_id_exact);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_path_is_stable_and_sanitized() {
|
||||
let je = journal_entry();
|
||||
@@ -645,8 +530,6 @@ mod tests {
|
||||
fn tier_delete_journal_paths_separate_legacy_and_backend_identities() {
|
||||
let mut legacy = journal_entry();
|
||||
legacy.backend_identity = None;
|
||||
legacy.version_id_exact = false;
|
||||
legacy.version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
let mut backend_a = journal_entry();
|
||||
backend_a.backend_identity = Some([1; 32]);
|
||||
let mut backend_b = journal_entry();
|
||||
@@ -692,8 +575,6 @@ mod tests {
|
||||
fn tier_delete_journal_without_transition_identity_stays_legacy() {
|
||||
let mut je = journal_entry();
|
||||
je.backend_identity = None;
|
||||
je.version_id_exact = false;
|
||||
je.version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
|
||||
let encoded = encode_tier_delete_journal_entry(&je).expect("legacy journal should remain encodable");
|
||||
let persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("journal JSON should decode");
|
||||
|
||||
@@ -185,7 +185,6 @@ struct ObjSweeper {
|
||||
transition_status: String,
|
||||
transition_tier: String,
|
||||
transition_version_id: String,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
remote_object: String,
|
||||
}
|
||||
|
||||
@@ -232,9 +231,7 @@ impl ObjSweeper {
|
||||
}
|
||||
|
||||
pub fn should_remove_remote_object(&self) -> Option<Jentry> {
|
||||
if self.transition_status != lifecycle::TRANSITION_COMPLETE
|
||||
|| self.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
if self.transition_status != lifecycle::TRANSITION_COMPLETE {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -252,11 +249,7 @@ impl ObjSweeper {
|
||||
version_id: self.transition_version_id.clone(),
|
||||
tier_name: self.transition_tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: matches!(
|
||||
self.transition_version_state,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: self.transition_version_state,
|
||||
version_id_exact: false,
|
||||
});
|
||||
}
|
||||
None
|
||||
@@ -293,7 +286,6 @@ pub struct Jentry {
|
||||
pub(crate) tier_name: String,
|
||||
pub(crate) backend_identity: Option<TierDestinationId>,
|
||||
pub(crate) version_id_exact: bool,
|
||||
pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
|
||||
}
|
||||
|
||||
impl ExpiryOp for Jentry {
|
||||
@@ -338,7 +330,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false).await
|
||||
}
|
||||
|
||||
async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
@@ -346,11 +338,8 @@ async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
validate_remote_version_id: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
if validate_remote_version_id {
|
||||
lease.validate_remote_version_id(rv_id)?;
|
||||
}
|
||||
lease.validate_remote_version_id(rv_id)?;
|
||||
|
||||
if remote_delete_breaker_is_open(Instant::now()).await {
|
||||
metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
|
||||
@@ -446,53 +435,7 @@ pub(crate) async fn delete_object_from_remote_tier_with_lease_idempotent(
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, version_id_exact, true).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idempotent(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
if rv_id.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"confirmed versioned transition candidate requires a non-empty remote version",
|
||||
));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if obj_name == "remote/empty-guard-probe" {
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, true, false).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
tier_name: &str,
|
||||
backend_identity: TierDestinationId,
|
||||
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, tier_name, backend_identity)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(obj_name, rv_id, &lease).await
|
||||
}
|
||||
|
||||
async fn delete_object_from_remote_tier_with_lease_idempotent_inner(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
validate_remote_version_id: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact, validate_remote_version_id)
|
||||
.await
|
||||
{
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact).await {
|
||||
Ok(()) => Ok(RemoteTierDeleteOutcome::Deleted),
|
||||
Err(err) if is_remote_tier_not_found_error(&err) => Ok(RemoteTierDeleteOutcome::AlreadyRemoved),
|
||||
Err(err) => {
|
||||
@@ -517,7 +460,6 @@ pub fn transitioned_delete_journal_entry(
|
||||
versioned: bool,
|
||||
suspended: bool,
|
||||
transitioned: &TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
let sweeper = ObjSweeper {
|
||||
version_id,
|
||||
@@ -526,7 +468,6 @@ pub fn transitioned_delete_journal_entry(
|
||||
transition_status: transitioned.status.clone(),
|
||||
transition_tier: transitioned.tier.clone(),
|
||||
transition_version_id: transitioned.version_id.clone(),
|
||||
transition_version_state,
|
||||
remote_object: transitioned.name.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -534,13 +475,8 @@ pub fn transitioned_delete_journal_entry(
|
||||
sweeper.should_remove_remote_object()
|
||||
}
|
||||
|
||||
pub fn transitioned_force_delete_journal_entry(
|
||||
transitioned: &TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
if transitioned.status != lifecycle::TRANSITION_COMPLETE
|
||||
|| transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject) -> Option<Jentry> {
|
||||
if transitioned.status != lifecycle::TRANSITION_COMPLETE {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -549,11 +485,7 @@ pub fn transitioned_force_delete_journal_entry(
|
||||
version_id: transitioned.version_id.clone(),
|
||||
tier_name: transitioned.tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: matches!(
|
||||
transition_version_state,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: transition_version_state,
|
||||
version_id_exact: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -562,14 +494,11 @@ mod test {
|
||||
use crate::client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, RemoteDeleteBreaker, RemoteTierDeleteOutcome,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure,
|
||||
};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -613,43 +542,6 @@ mod test {
|
||||
assert!(should_record_remote_delete_failure(&Error::other("NoSuchVersion")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transitioned_delete_journal_preserves_remote_version_state() {
|
||||
let cases = [
|
||||
(TransitionVersionState::Unknown, "legacy-version", None),
|
||||
(TransitionVersionState::KnownDisabled, "", Some(false)),
|
||||
(TransitionVersionState::SuspendedNull, "null", Some(true)),
|
||||
(TransitionVersionState::Exact, "opaque-version", Some(true)),
|
||||
];
|
||||
|
||||
for (state, version_id, expected_exact) in cases {
|
||||
let transitioned = TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: version_id.to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
status: lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let regular = transitioned_delete_journal_entry(None, false, false, &transitioned, state);
|
||||
let forced = transitioned_force_delete_journal_entry(&transitioned, state);
|
||||
|
||||
match expected_exact {
|
||||
Some(expected_exact) => {
|
||||
let regular = regular.expect("known version state should produce a regular delete journal entry");
|
||||
assert_eq!(regular.version_state, state);
|
||||
assert_eq!(regular.version_id_exact, expected_exact);
|
||||
let forced = forced.expect("known version state should produce a forced delete journal entry");
|
||||
assert_eq!(forced.version_state, state);
|
||||
assert_eq!(forced.version_id_exact, expected_exact);
|
||||
}
|
||||
None => {
|
||||
assert!(regular.is_none());
|
||||
assert!(forced.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn idempotent_remote_delete_treats_hooked_nosuchversion_as_already_removed() {
|
||||
@@ -772,55 +664,6 @@ mod test {
|
||||
assert_eq!(backend.remove_versions().await, vec![("remote/object".to_string(), String::new())]);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn confirmed_transition_cleanup_deletes_exact_provider_token() {
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
let manager = crate::services::tier::tier::TierConfigMgr::new();
|
||||
let backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await;
|
||||
let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM")
|
||||
.await
|
||||
.expect("test tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
drop(lease);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
|
||||
let outcome = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
"remote/object",
|
||||
"provider-version-token",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
.expect("confirmed upload compensation should delete the exact provider token");
|
||||
|
||||
assert_eq!(outcome, RemoteTierDeleteOutcome::Deleted);
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), "provider-version-token".to_string())]
|
||||
);
|
||||
|
||||
let err = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
"remote/empty-guard-probe",
|
||||
"",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
.expect_err("confirmed versioned cleanup must reject an empty token");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||
assert_eq!(backend.remove_count().await, 1);
|
||||
assert_eq!(
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.load(std::sync::atomic::Ordering::Relaxed),
|
||||
0,
|
||||
"empty remote versions must be rejected before exact cleanup dispatch"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breaker_opens_at_threshold_and_recovers_after_window() {
|
||||
let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
|
||||
|
||||
@@ -22,11 +22,7 @@ use uuid::Uuid;
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent_with_manager_and_identity;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -617,199 +613,6 @@ pub enum TransitionTransactionRecoveryOutcome {
|
||||
Retained,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TransitionOperatorProbe {
|
||||
Missing,
|
||||
UnversionedPresent,
|
||||
VersionedPresent(String),
|
||||
Ambiguous,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
|
||||
fn from(value: TransitionCandidateProbe) -> Self {
|
||||
match value {
|
||||
TransitionCandidateProbe::Missing => Self::Missing,
|
||||
TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
|
||||
TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
|
||||
TransitionCandidateProbe::Unsupported => Self::Unsupported,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionOperatorStatus {
|
||||
pub transaction_id: Uuid,
|
||||
pub state: TransitionTransactionState,
|
||||
pub tier_name: String,
|
||||
pub remote_object: String,
|
||||
pub not_after_unix_nanos: i64,
|
||||
pub probe: TransitionOperatorProbe,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionOperatorDeleteResult {
|
||||
pub status: TransitionOperatorStatus,
|
||||
pub journal_observed_after_delete: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TransitionOperatorError {
|
||||
#[error("transition transaction was not found")]
|
||||
NotFound,
|
||||
#[error("transition transaction is still inside its active ownership window")]
|
||||
NotExpired,
|
||||
#[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
|
||||
InvalidState(TransitionTransactionState),
|
||||
#[error("an exact non-empty remote version is required")]
|
||||
RemoteVersionRequired,
|
||||
#[error("remote candidate is not proven missing: {0:?}")]
|
||||
CandidateNotMissing(TransitionOperatorProbe),
|
||||
#[error("remote candidate version does not match requested exact version: expected {expected}, observed {actual:?}")]
|
||||
CandidateVersionMismatch {
|
||||
expected: String,
|
||||
actual: TransitionOperatorProbe,
|
||||
},
|
||||
#[error("transition transaction store failed: {0}")]
|
||||
Store(#[source] Error),
|
||||
#[error("remote tier reconciliation failed: {0}")]
|
||||
Remote(#[source] std::io::Error),
|
||||
}
|
||||
|
||||
type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
|
||||
|
||||
fn validate_operator_reconcile_transaction(
|
||||
transaction: &TransitionTransaction,
|
||||
now_unix_nanos: i128,
|
||||
) -> TransitionOperatorResult<()> {
|
||||
transaction
|
||||
.validate()
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
|
||||
return Err(TransitionOperatorError::InvalidState(transaction.state));
|
||||
}
|
||||
if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
|
||||
return Err(TransitionOperatorError::NotExpired);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_operator_reconcile_transaction(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<TransitionTransaction> {
|
||||
match load_transition_transaction_record(api, transaction_id).await {
|
||||
Ok(transaction) => Ok(transaction),
|
||||
Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
|
||||
Err(err) => Err(TransitionOperatorError::Store(err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn operator_probe_transition_candidate(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
) -> TransitionOperatorResult<TransitionOperatorProbe> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
|
||||
&api.tier_config_mgr(),
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
|
||||
lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
.map(TransitionOperatorProbe::from)
|
||||
.map_err(TransitionOperatorError::Remote)
|
||||
}
|
||||
|
||||
pub async fn inspect_transition_transaction_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<TransitionOperatorStatus> {
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let probe = operator_probe_transition_candidate(api, &transaction).await?;
|
||||
Ok(TransitionOperatorStatus {
|
||||
transaction_id,
|
||||
state: transaction.state,
|
||||
tier_name: transaction.tier_name,
|
||||
remote_object: transaction.remote_object,
|
||||
not_after_unix_nanos: transaction.not_after_unix_nanos,
|
||||
probe,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn delete_transition_candidate_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
remote_version_id: &str,
|
||||
) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
|
||||
if remote_version_id.is_empty() {
|
||||
return Err(TransitionOperatorError::RemoteVersionRequired);
|
||||
}
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
|
||||
&api.tier_config_mgr(),
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
|
||||
lease
|
||||
.validate_remote_version_id(remote_version_id)
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
let before_delete_probe = lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
.map(TransitionOperatorProbe::from)
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
if !matches!(&before_delete_probe, TransitionOperatorProbe::VersionedPresent(version_id) if version_id == remote_version_id) {
|
||||
return Err(TransitionOperatorError::CandidateVersionMismatch {
|
||||
expected: remote_version_id.to_string(),
|
||||
actual: before_delete_probe,
|
||||
});
|
||||
}
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
|
||||
let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
|
||||
Ok(_) => true,
|
||||
Err(Error::ConfigNotFound) => false,
|
||||
Err(err) => return Err(TransitionOperatorError::Store(err)),
|
||||
};
|
||||
Ok(TransitionOperatorDeleteResult {
|
||||
status: TransitionOperatorStatus {
|
||||
transaction_id,
|
||||
state: transaction.state,
|
||||
tier_name: transaction.tier_name,
|
||||
remote_object: transaction.remote_object,
|
||||
not_after_unix_nanos: transaction.not_after_unix_nanos,
|
||||
probe,
|
||||
},
|
||||
journal_observed_after_delete,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finalize_missing_transition_transaction_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<()> {
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
|
||||
if probe != TransitionOperatorProbe::Missing {
|
||||
return Err(TransitionOperatorError::CandidateNotMissing(probe));
|
||||
}
|
||||
delete_transition_transaction_record(api, transaction_id)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
|
||||
let transaction_id = transition_transaction_id_from_record_object_name(object)?;
|
||||
TransitionTransaction::decode(transaction_id, data)
|
||||
@@ -894,7 +697,7 @@ async fn recover_unknown_upload_outcome(
|
||||
.map_err(Error::other)?;
|
||||
|
||||
match lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.probe_transition_candidate(&transaction.remote_object)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
@@ -905,21 +708,6 @@ async fn recover_unknown_upload_outcome(
|
||||
TransitionCandidateProbe::UnversionedPresent => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
|
||||
{
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
&transaction.remote_object,
|
||||
&version_id,
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
&api.tier_config_mgr(),
|
||||
)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
|
||||
}
|
||||
@@ -1291,27 +1079,6 @@ mod tests {
|
||||
.expect("upload state change should succeed")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_reconcile_requires_expired_unknown_upload_outcome() {
|
||||
let mut transaction = new_transaction();
|
||||
let active_deadline = transaction.not_after_unix_nanos;
|
||||
|
||||
assert!(matches!(
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
|
||||
Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
|
||||
));
|
||||
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("unknown upload outcome should be recorded");
|
||||
assert!(matches!(
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
|
||||
Err(TransitionOperatorError::NotExpired)
|
||||
));
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
|
||||
.expect("expired unknown upload outcome should be eligible");
|
||||
}
|
||||
|
||||
fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
|
||||
TransitionCleanupProof {
|
||||
transaction_id: transaction.transaction_id,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -46,7 +46,7 @@ mod runtime_boundary;
|
||||
pub use datatypes::ResyncStatusType;
|
||||
pub use replication_config_boundary::{
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_target_arns,
|
||||
should_remove_replication_target, validate_replication_config_target_arns,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
|
||||
@@ -14,5 +14,5 @@
|
||||
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_target_arns,
|
||||
should_remove_replication_target, validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -14,11 +14,8 @@
|
||||
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use super::replication_error_boundary::Result;
|
||||
use super::replication_filemeta_boundary::{ReplicateDecision, ReplicationStatusType, ReplicationType};
|
||||
use super::replication_object_config::{
|
||||
check_replicate_delete, check_replicate_delete_strict, get_must_replicate_options, must_replicate,
|
||||
};
|
||||
use super::replication_object_config::{check_replicate_delete, get_must_replicate_options, must_replicate};
|
||||
use super::replication_object_decision_boundary::MustReplicateOptions;
|
||||
use super::replication_pool::{schedule_replication, schedule_replication_delete};
|
||||
use super::replication_queue_boundary::DeletedObjectReplicationInfo;
|
||||
@@ -53,16 +50,6 @@ impl ReplicationObjectBridge {
|
||||
check_replicate_delete(bucket, object, source, opts, get_error).await
|
||||
}
|
||||
|
||||
pub async fn check_delete_strict(
|
||||
bucket: &str,
|
||||
object: &ObjectToDelete,
|
||||
source: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
get_error: Option<String>,
|
||||
) -> Result<ReplicateDecision> {
|
||||
check_replicate_delete_strict(bucket, object, source, opts, get_error).await
|
||||
}
|
||||
|
||||
pub async fn schedule_object<S: ReplicationStorage>(
|
||||
object: ObjectInfo,
|
||||
storage: Arc<S>,
|
||||
|
||||
@@ -169,8 +169,9 @@ pub(crate) async fn check_replicate_delete(
|
||||
del_opts: &ObjectOptions,
|
||||
gerr: Option<String>,
|
||||
) -> ReplicateDecision {
|
||||
match check_replicate_delete_strict(bucket, dobj, oi, del_opts, gerr).await {
|
||||
Ok(decision) => decision,
|
||||
let rcfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => return ReplicateDecision::default(),
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
|
||||
@@ -181,30 +182,16 @@ pub(crate) async fn check_replicate_delete(
|
||||
error = %err,
|
||||
"Failed to look up replication config for delete replication"
|
||||
);
|
||||
ReplicateDecision::default()
|
||||
return ReplicateDecision::default();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn check_replicate_delete_strict(
|
||||
bucket: &str,
|
||||
dobj: &ObjectToDelete,
|
||||
oi: &ObjectInfo,
|
||||
del_opts: &ObjectOptions,
|
||||
gerr: Option<String>,
|
||||
) -> Result<ReplicateDecision> {
|
||||
let rcfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => return Ok(ReplicateDecision::default()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
if del_opts.replication_request {
|
||||
return Ok(ReplicateDecision::default());
|
||||
return ReplicateDecision::default();
|
||||
}
|
||||
|
||||
if !del_opts.versioned && !del_opts.version_suspended {
|
||||
return Ok(ReplicateDecision::default());
|
||||
if !del_opts.versioned {
|
||||
return ReplicateDecision::default();
|
||||
}
|
||||
|
||||
let replication_delete = object_to_delete_for_replication(dobj);
|
||||
@@ -222,7 +209,7 @@ pub(crate) async fn check_replicate_delete_strict(
|
||||
let mut dsc = ReplicateDecision::new();
|
||||
|
||||
if tgt_arns.is_empty() {
|
||||
return Ok(dsc);
|
||||
return dsc;
|
||||
}
|
||||
|
||||
for tgt_arn in tgt_arns {
|
||||
@@ -252,7 +239,7 @@ pub(crate) async fn check_replicate_delete_strict(
|
||||
dsc.set(tgt_dsc);
|
||||
}
|
||||
|
||||
Ok(dsc)
|
||||
dsc
|
||||
}
|
||||
|
||||
pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplicateOptions) -> ReplicateDecision {
|
||||
|
||||
@@ -1151,6 +1151,60 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
|
||||
dobj.delete_object.version_id
|
||||
};
|
||||
|
||||
let _rcfg = match get_replication_config(&bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_missing",
|
||||
"Skipping replication delete because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: dobj.delete_object.object_name.clone(),
|
||||
version_id,
|
||||
delete_marker: dobj.delete_object.delete_marker,
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason = "replication_config_lookup_failed",
|
||||
"Skipping replication delete because replication config lookup failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: dobj.delete_object.object_name.clone(),
|
||||
version_id,
|
||||
delete_marker: dobj.delete_object.delete_marker,
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if dobj.delete_object.delete_marker
|
||||
&& let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id
|
||||
{
|
||||
|
||||
@@ -172,7 +172,7 @@ impl ProviderVersionCapabilities {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
if version_id.is_empty() {
|
||||
return Err(Error::new(
|
||||
ErrorKind::InvalidData,
|
||||
|
||||
@@ -12,20 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
|
||||
use crate::cluster::rpc::http_auth::{
|
||||
RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
|
||||
RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
|
||||
};
|
||||
use crate::cluster::rpc::{
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
|
||||
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
|
||||
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use http::{Request as HttpRequest, Response as HttpResponse, Uri};
|
||||
use http::Uri;
|
||||
use rustfs_protos::{
|
||||
ChannelClass, create_new_channel, get_channel_for_class,
|
||||
proto_gen::node_service::{
|
||||
@@ -33,19 +24,9 @@ use rustfs_protos::{
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
},
|
||||
};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
error::Error,
|
||||
future::Future,
|
||||
io::ErrorKind,
|
||||
pin::Pin,
|
||||
sync::{LazyLock, Mutex},
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use std::{error::Error, io::ErrorKind};
|
||||
use tonic::{service::interceptor::InterceptedService, transport::Channel};
|
||||
use tower::Service;
|
||||
use tracing::debug;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
|
||||
|
||||
@@ -54,7 +35,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c
|
||||
pub async fn node_service_time_out_client(
|
||||
addr: &String,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
// Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
|
||||
// `_for_class` variant below (grpc-optimization P1).
|
||||
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
|
||||
@@ -63,14 +44,13 @@ pub async fn node_service_time_out_client(
|
||||
pub async fn heal_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => channel,
|
||||
None => create_new_channel(addr).await?,
|
||||
};
|
||||
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -79,14 +59,13 @@ pub async fn heal_control_time_out_client(
|
||||
pub async fn tier_mutation_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => channel,
|
||||
None => create_new_channel(addr).await?,
|
||||
};
|
||||
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -102,7 +81,7 @@ pub async fn node_service_time_out_client_for_class(
|
||||
addr: &String,
|
||||
interceptor: TonicInterceptor,
|
||||
class: ChannelClass,
|
||||
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match class {
|
||||
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
|
||||
@@ -117,7 +96,6 @@ pub async fn node_service_time_out_client_for_class(
|
||||
};
|
||||
|
||||
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -125,83 +103,14 @@ pub async fn node_service_time_out_client_for_class(
|
||||
|
||||
pub async fn node_service_time_out_client_no_auth(
|
||||
addr: &String,
|
||||
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
|
||||
}
|
||||
|
||||
/// The typed `tonic::Status` an internode RPC failure was converted from, if
|
||||
/// this error carries one.
|
||||
pub(crate) fn embedded_tonic_status(io_err: &std::io::Error) -> Option<&tonic::Status> {
|
||||
io_err.get_ref()?.downcast_ref::<RpcStatusError>().map(RpcStatusError::status)
|
||||
}
|
||||
|
||||
/// Decide whether a gRPC status reports a peer we cannot currently reach,
|
||||
/// rather than an application outcome from a live peer.
|
||||
///
|
||||
/// `Unavailable` is the one code that means "no service behind this channel":
|
||||
/// the client transport raises it when the connection is broken, and the
|
||||
/// server's own not-ready gates use it deliberately.
|
||||
///
|
||||
/// `Unknown` is the client transport's escape hatch for a cause it could not
|
||||
/// map to a code — tower's "Service was not ready: <cause>", an h2 error with
|
||||
/// no gRPC mapping. Our handlers never return it, so there its message is the
|
||||
/// only evidence available and the anchored needles decide.
|
||||
///
|
||||
/// Every other code is an answer from a live peer and is never a transport
|
||||
/// failure, whatever its message says. That distinction is the point of
|
||||
/// classifying by code: a peer relaying its own downstream trouble as
|
||||
/// `Internal("connection refused ...")`, or a handler interpolating a local
|
||||
/// `io::Error` into `Status::internal`, answered us perfectly well. Marking it
|
||||
/// offline over that text is the bug this classification replaces. Likewise a
|
||||
/// `Cancelled` "Timeout expired" from the per-RPC channel deadline means the
|
||||
/// peer is slow, not gone; gating it would turn load into a partition.
|
||||
pub(crate) fn is_network_like_status(status: &tonic::Status) -> bool {
|
||||
match status.code() {
|
||||
tonic::Code::Unavailable => true,
|
||||
tonic::Code::Unknown => message_has_network_needle(&status.to_string()),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Substring fallback for failures that only exist as text: dial errors
|
||||
/// wrapped by `get_client`, remote `error_info` payloads, and statuses
|
||||
/// flattened through `format!`. Needles must stay anchored to transport
|
||||
/// context — a bare word like "unavailable" also matches application text
|
||||
/// (e.g. a bucket named "unavailable-logs") and would take a healthy peer
|
||||
/// offline.
|
||||
pub(crate) fn message_has_network_needle(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
[
|
||||
"temporarily offline",
|
||||
"transport error",
|
||||
// tonic >= 0.14 renders Code::Unavailable as
|
||||
// `code: 'The service is currently unavailable'`.
|
||||
"code: 'the service is currently unavailable'",
|
||||
// RUSTFS_COMPAT_TODO(tonic-013-status-render): releases up to 1.0.0-alpha.38 shipped tonic 0.13, which rendered the same status as `status: Unavailable`, and peers relay that text in error_info. Remove after the minimum supported RustFS peer version ships tonic >= 0.14.
|
||||
"status: unavailable",
|
||||
"error trying to connect",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"broken pipe",
|
||||
"not connected",
|
||||
"unexpected eof",
|
||||
"timed out",
|
||||
"deadline has elapsed",
|
||||
"connection closed",
|
||||
"connection aborted",
|
||||
"tcp connect error",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| message.contains(needle))
|
||||
}
|
||||
|
||||
pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
||||
match err {
|
||||
DiskError::Timeout => true,
|
||||
DiskError::Io(io_err) => {
|
||||
if let Some(status) = embedded_tonic_status(io_err) {
|
||||
return is_network_like_status(status);
|
||||
}
|
||||
if matches!(
|
||||
io_err.kind(),
|
||||
ErrorKind::TimedOut
|
||||
@@ -215,110 +124,29 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
message_has_network_needle(&io_err.to_string())
|
||||
let message = io_err.to_string().to_ascii_lowercase();
|
||||
[
|
||||
"transport error",
|
||||
"unavailable",
|
||||
"error trying to connect",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"broken pipe",
|
||||
"not connected",
|
||||
"unexpected eof",
|
||||
"timed out",
|
||||
"deadline has elapsed",
|
||||
"connection closed",
|
||||
"connection aborted",
|
||||
"tcp connect error",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| message.contains(needle))
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The transport service that learns an authenticated peer boot epoch and adds the replay-scoped
|
||||
/// signature only after one has been observed. The v1/v2 interceptor stays inside this wrapper so
|
||||
/// old servers continue receiving precisely the metadata they understand.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ReplayScopeChannel<S> {
|
||||
inner: S,
|
||||
audience: Option<String>,
|
||||
}
|
||||
|
||||
/// The channel type used by internode clients after v2 authentication and replay-scope handling.
|
||||
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
|
||||
|
||||
static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
impl<S> ReplayScopeChannel<S> {
|
||||
fn new(inner: S, audience: Option<String>) -> Self {
|
||||
Self { inner, audience }
|
||||
}
|
||||
}
|
||||
|
||||
fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
|
||||
PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
|
||||
}
|
||||
|
||||
fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
|
||||
if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
|
||||
epochs.insert(audience, epoch);
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
|
||||
where
|
||||
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
|
||||
S::Error: Send + 'static,
|
||||
S::Future: Send + 'static,
|
||||
ReqBody: Send + 'static,
|
||||
ResBody: Send + 'static,
|
||||
{
|
||||
type Response = HttpResponse<ResBody>;
|
||||
type Error = S::Error;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
|
||||
let authenticated = self.audience.as_ref().is_some_and(|_| {
|
||||
request
|
||||
.headers()
|
||||
.get(RPC_AUTH_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
== Some(RPC_AUTH_VERSION_V2)
|
||||
});
|
||||
let challenge = authenticated.then(Uuid::new_v4);
|
||||
if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
|
||||
// The challenge is independently HMAC-authenticated by the response proof. It is not
|
||||
// part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
|
||||
request.headers_mut().insert(
|
||||
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
challenge.to_string().parse().expect("UUID must be a valid header value"),
|
||||
);
|
||||
if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
|
||||
cached_peer_boot_epoch(audience),
|
||||
request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
|
||||
request
|
||||
.headers()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
) {
|
||||
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
|
||||
Ok(headers) => request.headers_mut().extend(headers),
|
||||
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let audience = self.audience.clone();
|
||||
let future = self.inner.call(request);
|
||||
Box::pin(async move {
|
||||
let response = future.await?;
|
||||
if let (Some(audience), Some(challenge)) = (audience, challenge) {
|
||||
match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
|
||||
Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
|
||||
Err(error)
|
||||
if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
|
||||
|| response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
|
||||
{
|
||||
debug!(error = %error, "peer boot epoch response proof was rejected")
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TonicSignatureInterceptor {
|
||||
audience: Option<String>,
|
||||
}
|
||||
@@ -377,13 +205,6 @@ impl TonicInterceptor {
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
fn replay_scope_audience(&self) -> Option<String> {
|
||||
match self {
|
||||
Self::Signature(interceptor) => interceptor.audience.clone(),
|
||||
Self::NoOp(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl tonic::service::Interceptor for TonicInterceptor {
|
||||
@@ -406,38 +227,6 @@ mod tests {
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||
use tracing_subscriber::{Registry, layer::SubscriberExt};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct EpochProofService {
|
||||
audience: String,
|
||||
seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
|
||||
}
|
||||
|
||||
impl Service<HttpRequest<()>> for EpochProofService {
|
||||
type Response = HttpResponse<()>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
|
||||
self.seen_headers
|
||||
.lock()
|
||||
.expect("test header capture lock must not be poisoned")
|
||||
.push(request.headers().clone());
|
||||
let challenge = tonic_boot_epoch_challenge(request.headers())
|
||||
.expect("client challenge must be syntactically valid")
|
||||
.expect("authenticated client request must carry a boot epoch challenge");
|
||||
let mut response = HttpResponse::new(());
|
||||
response.headers_mut().extend(
|
||||
tonic_boot_epoch_response_headers(&self.audience, challenge)
|
||||
.expect("test server must be able to sign an epoch proof"),
|
||||
);
|
||||
std::future::ready(Ok(response))
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_test_rpc_secret() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
}
|
||||
@@ -480,70 +269,6 @@ mod tests {
|
||||
let _ = provider.shutdown();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_like_disk_error_uses_typed_status_code() {
|
||||
// Transport-level Unavailable statuses justify retry/eviction.
|
||||
assert!(is_network_like_disk_error(&DiskError::from(tonic::Status::unavailable(
|
||||
"storage layer is not initialized"
|
||||
))));
|
||||
// Application statuses from a live peer must not look network-like,
|
||||
// even when their message contains transport-sounding words.
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::internal(
|
||||
"failed to heal bucket \"unavailable-logs\""
|
||||
))));
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::unauthenticated(
|
||||
"No valid auth token"
|
||||
))));
|
||||
// A slow peer that blew the per-RPC deadline is still answering.
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::cancelled("Timeout expired"))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_tonic_status_is_recovered_across_error_conversions() {
|
||||
// DiskError and StorageError share one wrapper, so a status keeps its
|
||||
// typed classification whichever error it was converted into first.
|
||||
let from_storage: DiskErrorType = crate::error::Error::from(tonic::Status::unavailable("peer gone")).into();
|
||||
let DiskError::Io(io_err) = &from_storage else {
|
||||
panic!("status-derived disk error should stay an Io error");
|
||||
};
|
||||
assert_eq!(embedded_tonic_status(io_err).map(|status| status.code()), Some(tonic::Code::Unavailable));
|
||||
|
||||
let from_disk = crate::error::Error::from(DiskError::from(tonic::Status::unavailable("peer gone")));
|
||||
let crate::error::Error::Io(io_err) = &from_disk else {
|
||||
panic!("status-derived storage error should stay an Io error");
|
||||
};
|
||||
assert_eq!(embedded_tonic_status(io_err).map(|status| status.code()), Some(tonic::Code::Unavailable));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_like_disk_error_ignores_transport_words_in_application_statuses() {
|
||||
// Same contract as the peer client: a status the peer answered with
|
||||
// is not a transport failure, so it must not drive a reconnect even
|
||||
// when its message describes one.
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::internal(
|
||||
"connection refused while dialing downstream backend"
|
||||
))));
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::unauthenticated(
|
||||
"connection reset while validating token"
|
||||
))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_like_disk_error_requires_anchored_unavailable_needle() {
|
||||
// Regression: a bare "unavailable" needle used to match application
|
||||
// text such as a bucket name.
|
||||
assert!(!is_network_like_disk_error(&DiskError::other("bucket \"unavailable-logs\" not found")));
|
||||
// Anchored renderings of a flattened Unavailable status still match.
|
||||
assert!(is_network_like_disk_error(&DiskError::other(
|
||||
"code: 'The service is currently unavailable', message: \"peer gone\""
|
||||
)));
|
||||
assert!(is_network_like_disk_error(&DiskError::other(
|
||||
"status: Unavailable, message: \"peer gone\""
|
||||
)));
|
||||
assert!(is_network_like_disk_error(&DiskError::other("connection refused")));
|
||||
assert!(!is_network_like_disk_error(&DiskError::FileNotFound));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_keeps_auth_headers() {
|
||||
ensure_test_rpc_secret();
|
||||
@@ -579,52 +304,6 @@ mod tests {
|
||||
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "replay-scope-client-test:9000";
|
||||
PEER_BOOT_EPOCHS
|
||||
.lock()
|
||||
.expect("peer epoch cache lock must not be poisoned")
|
||||
.remove(audience);
|
||||
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let service = EpochProofService {
|
||||
audience: audience.to_string(),
|
||||
seen_headers: seen_headers.clone(),
|
||||
};
|
||||
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
|
||||
let make_request = || {
|
||||
let mut request = HttpRequest::builder()
|
||||
.uri("/node_service.NodeService/Ping")
|
||||
.body(())
|
||||
.expect("test RPC request must build");
|
||||
request.headers_mut().extend(
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 test headers must mint"),
|
||||
);
|
||||
request
|
||||
};
|
||||
|
||||
futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
|
||||
futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
|
||||
|
||||
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
|
||||
assert_eq!(headers.len(), 2);
|
||||
assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
|
||||
assert!(
|
||||
!headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||
"the first request must remain v2-compatible until the peer proves its epoch"
|
||||
);
|
||||
assert!(
|
||||
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||
"the second request must carry the replay-scoped v3 signature"
|
||||
);
|
||||
PEER_BOOT_EPOCHS
|
||||
.lock()
|
||||
.expect("peer epoch cache lock must not be poisoned")
|
||||
.remove(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_requires_generated_method_metadata() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -20,8 +20,8 @@
|
||||
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
|
||||
//! below, plus the broader negative-signature suite. The advisory class is: a
|
||||
//! node must never accept an RPC whose auth is missing, malformed, or signed
|
||||
//! with the default/empty shared secret. Body-bound v2 requests and all replay-scoped v3
|
||||
//! requests additionally receive process-local replay protection. See
|
||||
//! with the default/empty shared secret. Body-bound v2 requests additionally
|
||||
//! receive process-local replay protection. See
|
||||
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
|
||||
//!
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
|
||||
@@ -50,22 +50,13 @@ use uuid::Uuid;
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
|
||||
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||
pub(crate) const RPC_AUTH_VERSION_V2: &str = "2";
|
||||
pub const RPC_REPLAY_SCOPE_VERSION_HEADER: &str = "x-rustfs-rpc-replay-scope-version";
|
||||
pub const RPC_REPLAY_SCOPE_SIGNATURE_HEADER: &str = "x-rustfs-rpc-signature-v3";
|
||||
pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
|
||||
pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
|
||||
pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
|
||||
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
|
||||
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_AUTH_VERSION_V2: &str = "2";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
@@ -84,15 +75,9 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
)
|
||||
});
|
||||
static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
get_env_bool(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
)
|
||||
});
|
||||
// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
|
||||
// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
|
||||
// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
|
||||
// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
|
||||
// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
|
||||
// cannot disable replay protection by rejecting every body-bound request.
|
||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
@@ -101,7 +86,6 @@ static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
.max(1)
|
||||
});
|
||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||
static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcNonceCache {
|
||||
@@ -329,189 +313,6 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ReplayScope<'a> {
|
||||
audience: &'a str,
|
||||
path: &'a str,
|
||||
timestamp: &'a str,
|
||||
nonce: Uuid,
|
||||
content_sha256: &'a str,
|
||||
boot_epoch: Uuid,
|
||||
}
|
||||
|
||||
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
|
||||
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
|
||||
for part in [
|
||||
scope.audience.as_bytes(),
|
||||
b"|",
|
||||
scope.path.as_bytes(),
|
||||
b"|POST|",
|
||||
scope.timestamp.as_bytes(),
|
||||
b"|",
|
||||
scope.nonce.as_bytes(),
|
||||
b"|",
|
||||
scope.content_sha256.as_bytes(),
|
||||
b"|",
|
||||
scope.boot_epoch.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std::io::Result<String> {
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_replay_scope(&mut mac, scope);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
|
||||
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
|
||||
return false;
|
||||
};
|
||||
update_replay_scope(&mut mac, scope);
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
fn update_boot_epoch_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
|
||||
mac.update(RPC_BOOT_EPOCH_PROOF_DOMAIN);
|
||||
mac.update(audience.as_bytes());
|
||||
mac.update(b"|");
|
||||
mac.update(challenge.as_bytes());
|
||||
mac.update(boot_epoch.as_bytes());
|
||||
}
|
||||
|
||||
fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid) -> std::io::Result<String> {
|
||||
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
|
||||
}
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
|
||||
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
|
||||
}
|
||||
let proof = general_purpose::STANDARD
|
||||
.decode(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
mac.verify_slice(&proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
|
||||
}
|
||||
|
||||
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
|
||||
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
|
||||
(!value.is_nil())
|
||||
.then_some(value)
|
||||
.ok_or_else(|| std::io::Error::other(format!("Invalid {name}")))
|
||||
}
|
||||
|
||||
fn parse_tonic_rpc_path(path: &str) -> std::io::Result<(&str, &str)> {
|
||||
path.strip_prefix('/')
|
||||
.and_then(|path| path.split_once('/'))
|
||||
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
|
||||
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))
|
||||
}
|
||||
|
||||
/// The process-unique epoch included in every replay-scoped server verification.
|
||||
///
|
||||
/// A fresh process gets a fresh value, so a signature captured before a server restart cannot be
|
||||
/// admitted even though the bounded in-memory nonce cache necessarily starts empty again.
|
||||
pub fn tonic_rpc_boot_epoch() -> Uuid {
|
||||
*RPC_BOOT_EPOCH
|
||||
}
|
||||
|
||||
/// Build the additive replay-scope headers for a request that already carries rolling-upgrade-safe
|
||||
/// v1/v2 metadata. `timestamp` and `content_sha256` are deliberately reused from the v2 scope so
|
||||
/// old servers can continue validating the same request unchanged.
|
||||
pub fn gen_tonic_replay_scope_headers(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
timestamp: &str,
|
||||
content_sha256: &str,
|
||||
boot_epoch: Uuid,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
|
||||
}
|
||||
parse_tonic_rpc_path(path)?;
|
||||
timestamp
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
|
||||
let nonce = Uuid::new_v4();
|
||||
let signature = generate_replay_scope_signature(
|
||||
&get_shared_secret()?,
|
||||
ReplayScope {
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
nonce,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
},
|
||||
)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||
headers.insert(
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
header_value(&signature, RPC_REPLAY_SCOPE_SIGNATURE_HEADER)?,
|
||||
);
|
||||
headers.insert(
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
header_value(&nonce.to_string(), RPC_REPLAY_SCOPE_NONCE_HEADER)?,
|
||||
);
|
||||
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Parse the optional client challenge used to authenticate a server boot-epoch advertisement.
|
||||
pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option<Uuid>> {
|
||||
headers
|
||||
.get(RPC_BOOT_EPOCH_CHALLENGE_HEADER)
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch challenge"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC boot epoch challenge"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Build the authenticated response headers for a client boot-epoch challenge.
|
||||
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
|
||||
let boot_epoch = tonic_rpc_boot_epoch();
|
||||
let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
|
||||
headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Verify the server boot-epoch response for a challenge generated by this client.
|
||||
pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
|
||||
let boot_epoch = headers
|
||||
.get(RPC_BOOT_EPOCH_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
|
||||
let proof = headers
|
||||
.get(RPC_BOOT_EPOCH_PROOF_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
|
||||
verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
|
||||
Ok(boot_epoch)
|
||||
}
|
||||
|
||||
fn valid_content_sha256(value: &str) -> bool {
|
||||
value == UNSIGNED_PAYLOAD
|
||||
|| (value.len() == 64
|
||||
@@ -642,16 +443,6 @@ pub fn set_tonic_canonical_body_digest<T>(request: &mut tonic::Request<T>, canon
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_tonic_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(
|
||||
request: &mut tonic::Request<T>,
|
||||
) -> std::io::Result<()> {
|
||||
let canonical_body = request
|
||||
.get_ref()
|
||||
.canonical_body()
|
||||
.map_err(|_| std::io::Error::other("RPC mutation body length cannot be represented"))?;
|
||||
set_tonic_canonical_body_digest(request, &canonical_body)
|
||||
}
|
||||
|
||||
pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
let version = request
|
||||
.metadata()
|
||||
@@ -675,7 +466,7 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify a mutating RPC's canonical body digest with a rolling-upgrade fallback.
|
||||
/// Verify a mutating disk RPC's canonical body digest with a rolling-upgrade fallback.
|
||||
///
|
||||
/// When the request carries a real (non-`UNSIGNED-PAYLOAD`) content SHA-256 it is verified exactly
|
||||
/// like [`verify_tonic_canonical_body_digest`]. The digest value is a member of the signed v2
|
||||
@@ -706,7 +497,7 @@ fn verify_tonic_mutation_body_digest_with_strictness<T>(
|
||||
Some(digest) if digest != UNSIGNED_PAYLOAD => verify_tonic_canonical_body_digest(request, canonical_body),
|
||||
_ => {
|
||||
// RUSTFS_COMPAT_TODO(disk-mutation-body-digest): accept digestless peers during rolling upgrades. Remove after the
|
||||
// minimum supported RustFS peer version body-binds every mutating RPC.
|
||||
// minimum supported RustFS peer version body-binds every mutating disk RPC.
|
||||
if strict {
|
||||
return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
|
||||
}
|
||||
@@ -730,17 +521,6 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
|
||||
.any(|name| headers.contains_key(*name))
|
||||
}
|
||||
|
||||
fn has_replay_scope_headers(headers: &HeaderMap) -> bool {
|
||||
[
|
||||
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
RPC_BOOT_EPOCH_HEADER,
|
||||
]
|
||||
.iter()
|
||||
.any(|name| headers.contains_key(*name))
|
||||
}
|
||||
|
||||
/// Whether the server requires target-bound v2 authentication on every internode gRPC request,
|
||||
/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
|
||||
/// lever gated on the v1-fallback counter reading zero fleet-wide; see
|
||||
@@ -750,127 +530,9 @@ fn internode_rpc_signature_strict() -> bool {
|
||||
*INTERNODE_RPC_SIGNATURE_STRICT
|
||||
}
|
||||
|
||||
fn internode_rpc_replay_scope_strict() -> bool {
|
||||
*INTERNODE_RPC_REPLAY_SCOPE_STRICT
|
||||
}
|
||||
|
||||
fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
if audience.is_empty() {
|
||||
return Err(std::io::Error::other("Missing RPC audience"));
|
||||
}
|
||||
parse_tonic_rpc_path(path)?;
|
||||
|
||||
let version = headers
|
||||
.get(RPC_REPLAY_SCOPE_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope version"))?;
|
||||
if version != RPC_REPLAY_SCOPE_VERSION_V3 {
|
||||
return Err(std::io::Error::other("Unsupported RPC replay scope version"));
|
||||
}
|
||||
let signature = headers
|
||||
.get(RPC_REPLAY_SCOPE_SIGNATURE_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope signature"))?;
|
||||
let timestamp = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
|
||||
let signed_at = timestamp
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
check_timestamp(signed_at)?;
|
||||
let nonce = headers
|
||||
.get(RPC_REPLAY_SCOPE_NONCE_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope nonce"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC replay scope nonce"))?;
|
||||
let content_sha256 = headers
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
|
||||
if !valid_content_sha256(content_sha256) {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
let boot_epoch = headers
|
||||
.get(RPC_BOOT_EPOCH_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
|
||||
let secret = get_shared_secret()?;
|
||||
if !verify_replay_scope_signature(
|
||||
&secret,
|
||||
ReplayScope {
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
nonce,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
},
|
||||
signature,
|
||||
) {
|
||||
return Err(std::io::Error::other("Invalid RPC replay scope signature"));
|
||||
}
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_nonce(nonce, signed_at)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
verify_tonic_rpc_signature_with_policy(
|
||||
audience,
|
||||
path,
|
||||
headers,
|
||||
internode_rpc_signature_strict(),
|
||||
internode_rpc_replay_scope_strict(),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication while allowing the narrowly scoped v2 `Ping` bootstrap used to
|
||||
/// obtain an authenticated server boot epoch when replay-scope strictness is enabled.
|
||||
pub fn verify_tonic_rpc_signature_with_bootstrap(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
headers: &HeaderMap,
|
||||
allow_replay_scope_bootstrap: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_tonic_rpc_signature_with_policy(
|
||||
audience,
|
||||
path,
|
||||
headers,
|
||||
internode_rpc_signature_strict(),
|
||||
internode_rpc_replay_scope_strict(),
|
||||
allow_replay_scope_bootstrap,
|
||||
)
|
||||
}
|
||||
|
||||
fn verify_tonic_rpc_signature_with_policy(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
headers: &HeaderMap,
|
||||
signature_strict: bool,
|
||||
replay_scope_strict: bool,
|
||||
allow_replay_scope_bootstrap: bool,
|
||||
) -> std::io::Result<()> {
|
||||
if has_replay_scope_headers(headers) {
|
||||
return verify_tonic_replay_scope_signature(audience, path, headers);
|
||||
}
|
||||
|
||||
// Only a method-bound v2 Ping with a syntactically valid challenge may bootstrap a strict
|
||||
// client after its peer restarts. Legacy metadata never gets this exception.
|
||||
let bootstrap = allow_replay_scope_bootstrap
|
||||
&& has_v2_auth_headers(headers)
|
||||
&& tonic_boot_epoch_challenge(headers).is_ok_and(|challenge| challenge.is_some());
|
||||
if replay_scope_strict && !bootstrap {
|
||||
return Err(std::io::Error::other("RPC replay-scoped authentication required"));
|
||||
}
|
||||
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, signature_strict)?;
|
||||
global_internode_metrics().record_replay_scope_fallback();
|
||||
Ok(())
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
|
||||
}
|
||||
|
||||
/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
|
||||
@@ -1015,28 +677,11 @@ mod tests {
|
||||
use crate::cluster::rpc::context_propagation::REQUEST_ID_HEADER;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use http::{HeaderMap, Method};
|
||||
use rustfs_protos::{
|
||||
CanonicalMutationBody as _, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
proto_gen::node_service::{Mss, SignalServiceRequest},
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::io::{self, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use time::OffsetDateTime;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
fn signal_service_request(signal: &str, sub_system: &str, dry_run: &str) -> SignalServiceRequest {
|
||||
SignalServiceRequest {
|
||||
vars: Some(Mss {
|
||||
value: HashMap::from([
|
||||
(PEER_RESTSIGNAL.to_string(), signal.to_string()),
|
||||
(PEER_RESTSUB_SYS.to_string(), sub_system.to_string()),
|
||||
(PEER_RESTDRY_RUN.to_string(), dry_run.to_string()),
|
||||
]),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
@@ -1601,104 +1246,6 @@ mod tests {
|
||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_binds_path_epoch_and_random_nonce() {
|
||||
ensure_test_rpc_secret();
|
||||
let path = "/node_service.NodeService/Ping";
|
||||
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 compatibility headers should build");
|
||||
let timestamp = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 timestamp")
|
||||
.to_string();
|
||||
let content_sha256 = headers
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 content digest")
|
||||
.to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers("node-a:9000", path, ×tamp, &content_sha256, tonic_rpc_boot_epoch())
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
|
||||
assert!(
|
||||
verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false).is_ok(),
|
||||
"the first replay-scoped request must be accepted"
|
||||
);
|
||||
let replay = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false)
|
||||
.expect_err("the random replay-scope nonce must be single-use");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
|
||||
let path_error = verify_tonic_replay_scope_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
|
||||
.expect_err("a replay-scoped signature must not move to another method");
|
||||
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||
ensure_test_rpc_secret();
|
||||
let path = "/node_service.NodeService/Ping";
|
||||
let mut partial = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 compatibility headers should build");
|
||||
partial.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||
let error = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
|
||||
.expect_err("partial replay-scope metadata must never downgrade to v2");
|
||||
assert_eq!(error.to_string(), "Missing RPC replay scope signature");
|
||||
|
||||
let timestamp = partial
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 timestamp")
|
||||
.to_string();
|
||||
let content_sha256 = partial
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 content digest")
|
||||
.to_string();
|
||||
let stale_epoch = Uuid::new_v4();
|
||||
partial.extend(
|
||||
gen_tonic_replay_scope_headers("node-a:9000", path, ×tamp, &content_sha256, stale_epoch)
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
let stale = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
|
||||
.expect_err("a signature from a prior server boot epoch must be rejected");
|
||||
assert_eq!(stale.to_string(), "RPC boot epoch is stale");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_strictness_allows_only_authenticated_ping_bootstrap() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 compatibility headers should build");
|
||||
let rejected =
|
||||
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, false)
|
||||
.expect_err("strict replay scope must reject stripped new metadata");
|
||||
assert_eq!(rejected.to_string(), "RPC replay-scoped authentication required");
|
||||
|
||||
headers.insert(
|
||||
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
HeaderValue::from_str(&Uuid::new_v4().to_string()).expect("UUID header"),
|
||||
);
|
||||
assert!(
|
||||
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, true,)
|
||||
.is_ok(),
|
||||
"only the signed Ping bootstrap may obtain a new server epoch in strict mode"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn boot_epoch_response_proof_binds_audience_challenge_and_epoch() {
|
||||
ensure_test_rpc_secret();
|
||||
let challenge = Uuid::new_v4();
|
||||
let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("proof headers should build");
|
||||
let epoch =
|
||||
verify_tonic_boot_epoch_response("node-a:9000", challenge, &headers).expect("matching proof headers should verify");
|
||||
assert_eq!(epoch, tonic_rpc_boot_epoch());
|
||||
assert!(verify_tonic_boot_epoch_response("node-b:9000", challenge, &headers).is_err());
|
||||
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
|
||||
ensure_test_rpc_secret();
|
||||
@@ -2049,72 +1596,6 @@ mod tests {
|
||||
assert_eq!(stripped.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_service_mutation_contract_rejects_tampering_and_replay() {
|
||||
ensure_test_rpc_secret();
|
||||
let body = signal_service_request("2", "scanner", "false")
|
||||
.canonical_body()
|
||||
.expect("small signal request should encode");
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("canonical body digest should be attached");
|
||||
let content_sha256 = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "SignalService", content_sha256)
|
||||
.expect("body-bound auth headers should build");
|
||||
request.metadata_mut().as_mut().extend(headers.clone());
|
||||
|
||||
assert!(
|
||||
verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers).is_ok(),
|
||||
"the first body-bound signal request must authenticate"
|
||||
);
|
||||
assert!(verify_tonic_mutation_body_digest(&request, &body).is_ok());
|
||||
|
||||
let tampered = signal_service_request("1", "scanner", "false")
|
||||
.canonical_body()
|
||||
.expect("small signal request should encode");
|
||||
let error = verify_tonic_mutation_body_digest(&request, &tampered)
|
||||
.expect_err("changing the signal must invalidate the signed digest");
|
||||
assert_eq!(error.to_string(), "RPC content SHA-256 mismatch");
|
||||
|
||||
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
|
||||
.expect_err("reusing the signal nonce must fail");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(rpc_body_digest_fallback_counter)]
|
||||
fn signal_service_mutation_contract_preserves_rollout_fallback_and_strictness() {
|
||||
let body = signal_service_request("2", "scanner", "false")
|
||||
.canonical_body()
|
||||
.expect("small signal request should encode");
|
||||
let before = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
let digestless = tonic::Request::new(());
|
||||
|
||||
assert!(
|
||||
verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, false).is_ok(),
|
||||
"old peers must remain compatible while the rollout gate is open"
|
||||
);
|
||||
assert_eq!(
|
||||
global_internode_metrics().snapshot().body_digest_fallback_total,
|
||||
before + 1,
|
||||
"accepted digestless signal requests must be visible in the fallback metric"
|
||||
);
|
||||
|
||||
let error = verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, true)
|
||||
.expect_err("strict mode must reject a digestless signal request");
|
||||
assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
|
||||
|
||||
let mut bound = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut bound, &body).expect("canonical body digest should be attached");
|
||||
bound
|
||||
.metadata_mut()
|
||||
.as_mut()
|
||||
.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
assert!(verify_tonic_mutation_body_digest_with_strictness(&bound, &body, true).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_rejects_replay_after_wall_clock_regression() {
|
||||
let now = Instant::now();
|
||||
|
||||
@@ -26,18 +26,13 @@ pub(crate) mod runtime_sources;
|
||||
pub use background_monitor::shutdown_background_monitors;
|
||||
pub(crate) use background_monitor::spawn_background_monitor;
|
||||
pub use client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth,
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
};
|
||||
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||
#[allow(unused_imports)]
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -13,10 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, heal_control_time_out_client, node_service_time_out_client,
|
||||
tier_mutation_control_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
@@ -50,7 +50,6 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
|
||||
use rustfs_utils::XHost;
|
||||
use serde::{Deserialize, Serialize as _};
|
||||
@@ -66,9 +65,13 @@ use std::{
|
||||
use tokio::{net::TcpStream, time::Duration};
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const PEER_RESTSIGNAL: &str = "signal";
|
||||
pub const PEER_RESTSUB_SYS: &str = "sub-sys";
|
||||
pub const PEER_RESTDRY_RUN: &str = "dry-run";
|
||||
pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
|
||||
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
|
||||
const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
|
||||
@@ -221,21 +224,6 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
|
||||
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
|
||||
}
|
||||
|
||||
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
|
||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
|
||||
if topology_member != expected_member {
|
||||
return Err(Error::other(
|
||||
"peer returned a remote version state capability for a different topology member",
|
||||
));
|
||||
}
|
||||
let server_epoch =
|
||||
Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
|
||||
if server_epoch.is_nil() {
|
||||
return Err(Error::other("peer returned a nil remote version state epoch"));
|
||||
}
|
||||
Ok(server_epoch)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
pub events: Vec<u8>,
|
||||
@@ -247,7 +235,6 @@ pub struct PeerLiveEventsBatch {
|
||||
pub struct PeerRestClient {
|
||||
pub host: XHost,
|
||||
pub grid_host: String,
|
||||
topology_member: String,
|
||||
offline: Arc<AtomicBool>,
|
||||
recovery_running: Arc<AtomicBool>,
|
||||
}
|
||||
@@ -340,11 +327,9 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
pub fn new(host: XHost, grid_host: String) -> Self {
|
||||
let topology_member = host.to_string();
|
||||
Self {
|
||||
host,
|
||||
grid_host,
|
||||
topology_member,
|
||||
offline: Arc::new(AtomicBool::new(false)),
|
||||
recovery_running: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
@@ -364,11 +349,7 @@ impl PeerRestClient {
|
||||
|
||||
let client = match grid_host {
|
||||
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
|
||||
Ok(host) => {
|
||||
let mut client = PeerRestClient::new(host, grid_host);
|
||||
client.topology_member = peer_host_port.clone();
|
||||
Some(client)
|
||||
}
|
||||
Ok(host) => Some(PeerRestClient::new(host, grid_host)),
|
||||
Err(err) => {
|
||||
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
|
||||
None
|
||||
@@ -411,7 +392,7 @@ impl PeerRestClient {
|
||||
(remote, all, remote_topology_hosts)
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
self.mark_offline_and_spawn_recovery();
|
||||
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
||||
@@ -432,7 +413,7 @@ impl PeerRestClient {
|
||||
&self,
|
||||
) -> Result<
|
||||
rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
|
||||
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||
InterceptedService<Channel, TonicInterceptor>,
|
||||
>,
|
||||
> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
@@ -453,7 +434,7 @@ impl PeerRestClient {
|
||||
|
||||
async fn get_tier_mutation_control_client(
|
||||
&self,
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
self.mark_offline_and_spawn_recovery();
|
||||
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
||||
@@ -490,21 +471,26 @@ impl PeerRestClient {
|
||||
self.offline.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Whether this failure means the peer is unreachable, so it should be
|
||||
/// gated offline and its connection evicted.
|
||||
///
|
||||
/// RPC failures are classified by their typed gRPC code first
|
||||
/// (`is_network_like_status`); an application error from a live peer must
|
||||
/// never take it offline no matter what its message says. The substring
|
||||
/// fallback only covers failures that exist purely as text, such as the
|
||||
/// dial errors `get_client` wraps.
|
||||
fn is_network_like_error(err: &Error) -> bool {
|
||||
if let Error::Io(io_err) = err
|
||||
&& let Some(status) = embedded_tonic_status(io_err)
|
||||
{
|
||||
return is_network_like_status(status);
|
||||
}
|
||||
message_has_network_needle(&err.to_string())
|
||||
let message = err.to_string().to_ascii_lowercase();
|
||||
[
|
||||
"temporarily offline",
|
||||
"transport error",
|
||||
"unavailable",
|
||||
"error trying to connect",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"broken pipe",
|
||||
"not connected",
|
||||
"unexpected eof",
|
||||
"timed out",
|
||||
"deadline has elapsed",
|
||||
"connection closed",
|
||||
"connection aborted",
|
||||
"tcp connect error",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| message.contains(needle))
|
||||
}
|
||||
|
||||
fn mark_offline_and_spawn_recovery(&self) {
|
||||
@@ -1175,24 +1161,14 @@ impl PeerRestClient {
|
||||
validate_heal_control_capability_proof(&canonical_ack, &proof)
|
||||
}
|
||||
|
||||
pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadBucketMetadataRequest {
|
||||
let request = Request::new(LoadBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
scanner_maintenance_change,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_bucket_metadata(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1212,10 +1188,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DeleteBucketMetadataRequest {
|
||||
let request = Request::new(DeleteBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_bucket_metadata(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1235,10 +1210,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DeletePolicyRequest {
|
||||
let request = Request::new(DeletePolicyRequest {
|
||||
policy_name: policy.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_policy(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1258,10 +1232,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadPolicyRequest {
|
||||
let request = Request::new(LoadPolicyRequest {
|
||||
policy_name: policy.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_policy(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1281,12 +1254,11 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadPolicyMappingRequest {
|
||||
let request = Request::new(LoadPolicyMappingRequest {
|
||||
user_or_group: user_or_group.to_string(),
|
||||
user_type,
|
||||
is_group,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_policy_mapping(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1306,10 +1278,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DeleteUserRequest {
|
||||
let request = Request::new(DeleteUserRequest {
|
||||
access_key: access_key.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_user(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1329,10 +1300,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DeleteServiceAccountRequest {
|
||||
let request = Request::new(DeleteServiceAccountRequest {
|
||||
access_key: access_key.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_service_account(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1352,11 +1322,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadUserRequest {
|
||||
let request = Request::new(LoadUserRequest {
|
||||
access_key: access_key.to_string(),
|
||||
temp,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_user(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1376,10 +1345,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadServiceAccountRequest {
|
||||
let request = Request::new(LoadServiceAccountRequest {
|
||||
access_key: access_key.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_service_account(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1399,10 +1367,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadGroupRequest {
|
||||
let request = Request::new(LoadGroupRequest {
|
||||
group: group.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_group(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1422,8 +1389,7 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(ReloadSiteReplicationConfigRequest {});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(ReloadSiteReplicationConfigRequest {});
|
||||
|
||||
let response = client.reload_site_replication_config(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1447,10 +1413,9 @@ impl PeerRestClient {
|
||||
vars.insert(PEER_RESTSIGNAL.to_string(), sig.to_string());
|
||||
vars.insert(PEER_RESTSUB_SYS.to_string(), sub_sys.to_string());
|
||||
vars.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string());
|
||||
let mut request = Request::new(SignalServiceRequest {
|
||||
let request = Request::new(SignalServiceRequest {
|
||||
vars: Some(Mss { value: vars }),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.signal_service(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1519,8 +1484,7 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(ReloadPoolMetaRequest {});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(ReloadPoolMetaRequest {});
|
||||
|
||||
let response = client.reload_pool_meta(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1541,10 +1505,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(StopRebalanceRequest {
|
||||
let request = Request::new(StopRebalanceRequest {
|
||||
expected_rebalance_id: expected_rebalance_id.unwrap_or_default().to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.stop_rebalance(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1565,8 +1528,7 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
|
||||
|
||||
let response = client.load_rebalance_meta(request).await?.into_inner();
|
||||
|
||||
@@ -1605,8 +1567,7 @@ impl PeerRestClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(StartDecommissionRequest { pool_indices });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(StartDecommissionRequest { pool_indices });
|
||||
|
||||
let response = client.start_decommission(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1629,8 +1590,7 @@ impl PeerRestClient {
|
||||
let pool_index = u32::try_from(pool_index)
|
||||
.map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(CancelDecommissionRequest { pool_index });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(CancelDecommissionRequest { pool_index });
|
||||
|
||||
let response = client.cancel_decommission(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1653,8 +1613,7 @@ impl PeerRestClient {
|
||||
let pool_index = u32::try_from(pool_index)
|
||||
.map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(ClearDecommissionRequest { pool_index });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(ClearDecommissionRequest { pool_index });
|
||||
|
||||
let response = client.clear_decommission(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1674,14 +1633,10 @@ impl PeerRestClient {
|
||||
pub async fn load_transition_tier_config(&self) -> Result<()> {
|
||||
match self.load_transition_tier_config_outcome().await {
|
||||
TierConfigReloadOutcome::Success => Ok(()),
|
||||
// Only a reconnect-class failure says anything about the channel.
|
||||
// `finalize_result` marks the peer offline and evicts its connection
|
||||
// whenever the message looks network-like, and a peer that answered
|
||||
// and rejected the apply can easily report one ("release RPC failed:
|
||||
// transport error"). Routing those through here would gate a healthy,
|
||||
// responding peer out of every unrelated RPC.
|
||||
TierConfigReloadOutcome::TransientReconnect(err) => self.finalize_result(Err(err)).await,
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(err) | TierConfigReloadOutcome::Terminal(err) => Err(err),
|
||||
TierConfigReloadOutcome::TransientReconnect(err) | TierConfigReloadOutcome::TransientRetrySameChannel(err) => {
|
||||
self.finalize_result(Err(err)).await
|
||||
}
|
||||
TierConfigReloadOutcome::Terminal(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1707,9 +1662,6 @@ impl PeerRestClient {
|
||||
Err(err) => return tier_config_reload_connection_outcome(err),
|
||||
};
|
||||
let mut request = Request::new(LoadTransitionTierConfigRequest {});
|
||||
if let Err(err) = set_tonic_mutation_body_digest(&mut request) {
|
||||
return TierConfigReloadOutcome::Terminal(Error::other(err));
|
||||
}
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
|
||||
let response = match client.load_transition_tier_config(request).await {
|
||||
@@ -1750,37 +1702,39 @@ fn tier_config_reload_connection_outcome(err: Error) -> TierConfigReloadOutcome
|
||||
}
|
||||
|
||||
fn is_tier_config_reload_connection_failure(err: &Error) -> bool {
|
||||
let message = err.to_string();
|
||||
// A bare "unavailable" is only trusted inside the local dial-failure
|
||||
// wrapper from `get_client`, never in application text.
|
||||
let message = err.to_string().to_ascii_lowercase();
|
||||
if message
|
||||
.to_ascii_lowercase()
|
||||
.split_once("can not get client, err:")
|
||||
.is_some_and(|(_, local_error)| local_error.contains("unavailable"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
message_has_network_needle(&message)
|
||||
[
|
||||
"temporarily offline",
|
||||
"transport error",
|
||||
"error trying to connect",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"connection closed",
|
||||
"connection aborted",
|
||||
"broken pipe",
|
||||
"not connected",
|
||||
"unexpected eof",
|
||||
"timed out",
|
||||
"deadline has elapsed",
|
||||
"tcp connect error",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| message.contains(needle))
|
||||
}
|
||||
|
||||
/// Classifies a reload the peer answered but refused to apply.
|
||||
///
|
||||
/// The peer replied, so the channel is healthy and only the remote apply
|
||||
/// failed. Those failures are transient by nature: the reload reads the tier
|
||||
/// mutation intents and takes the distributed tier-config lock, both of which
|
||||
/// fail while any other node is restarting or while the lock quorum is briefly
|
||||
/// disturbed. Retiring the worker on the first such rejection leaves that peer
|
||||
/// pinned to the old configuration with nothing left to heal it, so it answers
|
||||
/// `TierNotFound` for a tier the rest of the cluster already committed until a
|
||||
/// second admin mutation happens to spawn a fresh worker.
|
||||
///
|
||||
/// Convergence is the whole point of this path, so a rejection is retried on
|
||||
/// the same channel. The worker's exponential backoff caps the cost at one
|
||||
/// reload every `TIER_CONFIG_RELOAD_RETRY_CAP`, and `Terminal` stays reachable
|
||||
/// for transport and gRPC status failures, which is where a genuinely
|
||||
/// unrecoverable peer surfaces.
|
||||
fn tier_config_reload_remote_failure(error_info: Option<String>) -> TierConfigReloadOutcome {
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info.unwrap_or_default()))
|
||||
let error_info = error_info.unwrap_or_default();
|
||||
if matches!(error_info.as_str(), "errServerNotInitialized" | "ServerNotInitialized") {
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info))
|
||||
} else {
|
||||
TierConfigReloadOutcome::Terminal(Error::other(error_info))
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadOutcome {
|
||||
@@ -1790,14 +1744,6 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
|
||||
TierConfigReloadOutcome::TransientReconnect(status.into())
|
||||
} else if status.code() == Code::Unknown && status.message().starts_with("Service was not ready:") {
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(status.into())
|
||||
} else if status.code() == Code::Unknown
|
||||
&& is_tier_config_reload_connection_failure(&Error::other(status.message().to_string()))
|
||||
{
|
||||
// tonic reports a connection dropped mid-call as `Unknown` carrying the
|
||||
// transport error text rather than as `Unavailable`, which is what a peer
|
||||
// restarting under an active mutation produces. Reconnect and retry, so
|
||||
// the restart does not permanently retire this peer's reload worker.
|
||||
TierConfigReloadOutcome::TransientReconnect(status.into())
|
||||
} else {
|
||||
TierConfigReloadOutcome::Terminal(status.into())
|
||||
}
|
||||
@@ -2205,126 +2151,6 @@ mod tests {
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::NotImplemented));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_uses_typed_status_code() {
|
||||
// The one code that means "nothing is answering on this channel".
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::from(tonic::Status::unavailable(
|
||||
"storage layer is not initialized"
|
||||
))));
|
||||
// Application statuses from a live peer must not mark it offline,
|
||||
// even when their message contains transport-sounding words.
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(tonic::Status::internal(
|
||||
"failed to reload metadata for bucket \"unavailable-logs\""
|
||||
))));
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(tonic::Status::unauthenticated(
|
||||
"No valid auth token"
|
||||
))));
|
||||
// A request-budget expiry answered by a live peer is an application
|
||||
// outcome, not a transport failure.
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(tonic::Status::deadline_exceeded(
|
||||
"heal control request expired"
|
||||
))));
|
||||
// Unknown is the transport's escape hatch for a cause it could not
|
||||
// map, and our handlers never return it, so there the text decides.
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::from(tonic::Status::unknown(
|
||||
"Service was not ready: transport error"
|
||||
))));
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(tonic::Status::unknown(
|
||||
"peer response unknown"
|
||||
))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_ignores_transport_words_in_application_statuses() {
|
||||
// The reason classification reads the code rather than the text: a
|
||||
// peer that answers is reachable, even when what it says describes a
|
||||
// connection failure of its own. A handler interpolating a local
|
||||
// io::Error into Status::internal, or relaying trouble with its own
|
||||
// downstream, must not cost us the channel to a healthy peer.
|
||||
for status in [
|
||||
tonic::Status::internal("connection refused while dialing downstream backend"),
|
||||
tonic::Status::internal("write failed: broken pipe"),
|
||||
tonic::Status::unauthenticated("connection reset while validating token"),
|
||||
tonic::Status::failed_precondition("scanner lease timed out"),
|
||||
tonic::Status::deadline_exceeded("heal control request timed out"),
|
||||
] {
|
||||
let rendered = status.to_string();
|
||||
assert!(
|
||||
!PeerRestClient::is_network_like_error(&Error::from(status)),
|
||||
"an answered application status must not mark the peer offline: {rendered}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_keeps_slow_peers_online() {
|
||||
// The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s)
|
||||
// surfaces as Cancelled "Timeout expired" carrying the transport
|
||||
// cause as its source. A peer that is merely slow must stay online:
|
||||
// gating it would spend a full recovery cycle fast-failing every RPC
|
||||
// to a host that is still answering, turning load into a partition.
|
||||
let timeout_status = tonic::Status::cancelled("Timeout expired");
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(timeout_status)));
|
||||
|
||||
let sourced = tonic::Status::from_error(Box::new(std::io::Error::other("Timeout expired")));
|
||||
assert!(
|
||||
std::error::Error::source(&sourced).is_some(),
|
||||
"the transport builds this status through Status::from_error, which attaches the cause"
|
||||
);
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(sourced)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rpc_status_errors_keep_their_rendering_and_hide_peer_metadata() {
|
||||
let err = Error::from(tonic::Status::unavailable("peer gone"));
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
"Io error: code: 'The service is currently unavailable', message: \"peer gone\""
|
||||
);
|
||||
|
||||
// tonic's own Debug prints the MetadataMap, i.e. every response header
|
||||
// the peer sent; those must not reach a log through this error.
|
||||
let mut status = tonic::Status::unavailable("peer gone");
|
||||
status
|
||||
.metadata_mut()
|
||||
.insert("authorization", "Bearer secret".parse().expect("valid header value"));
|
||||
let rendered = format!("{:?}", Error::from(status));
|
||||
assert!(!rendered.contains("Bearer secret"), "{rendered}");
|
||||
assert!(!rendered.contains("MetadataMap"), "{rendered}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_ignores_application_text_containing_unavailable() {
|
||||
// Regression: a bare "unavailable" needle used to match application
|
||||
// strings like these and take a healthy peer offline.
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::other(
|
||||
"peer replication statistics provider is unavailable"
|
||||
)));
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::other(
|
||||
"bucket \"unavailable-logs\" not found"
|
||||
)));
|
||||
// Anchored renderings of a flattened Unavailable status still match:
|
||||
// tonic >= 0.14 form ...
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::other(
|
||||
"peer tier mutation commit RPC failed: code: 'The service is currently unavailable', message: \"peer gone\""
|
||||
)));
|
||||
// ... which is only anchored as long as tonic renders Unavailable this
|
||||
// way. A tonic bump that reworded it leaves the typed path correct but
|
||||
// this needle stale, so pin the coupling rather than discover it in a
|
||||
// partition.
|
||||
assert!(
|
||||
tonic::Status::unavailable("peer gone")
|
||||
.to_string()
|
||||
.to_ascii_lowercase()
|
||||
.contains("code: 'the service is currently unavailable'"),
|
||||
"tonic reworded Code::Unavailable; update the anchored needle"
|
||||
);
|
||||
// ... and the tonic <= 0.13 form peers may relay in error_info.
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::other(
|
||||
"peer tier mutation commit RPC failed: status: Unavailable, message: \"peer gone\""
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_config_reload_outcome_keeps_tonic_and_remote_errors_typed() {
|
||||
assert!(matches!(
|
||||
@@ -2351,12 +2177,9 @@ mod tests {
|
||||
tier_config_reload_status_outcome(tonic::Status::cancelled("request cancelled")),
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
// A peer that answered and then refused the apply is retried rather than
|
||||
// retired: the channel is healthy, so the rejection reflects remote state
|
||||
// that the next attempt can find healed.
|
||||
assert!(matches!(
|
||||
tier_config_reload_remote_failure(Some("backend unavailable".to_string())),
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(_)
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
tier_config_reload_remote_failure(Some("errServerNotInitialized".to_string())),
|
||||
@@ -2370,58 +2193,6 @@ mod tests {
|
||||
tier_config_reload_connection_outcome(Error::other("can not get client, err: connection unavailable")),
|
||||
TierConfigReloadOutcome::TransientReconnect(_)
|
||||
));
|
||||
// The bare word is trusted only to the right of the dial-failure
|
||||
// prefix, not anywhere in the message.
|
||||
assert!(matches!(
|
||||
tier_config_reload_connection_outcome(Error::other(
|
||||
"bucket unavailable-logs rejected it, then: can not get client, err: some other reason"
|
||||
)),
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
}
|
||||
|
||||
/// A tier mutation issued while another node restarts must still converge on
|
||||
/// the nodes that stayed up. Those peers answer the reload RPC and reject the
|
||||
/// apply, because reloading reads the tier mutation intents and takes the
|
||||
/// distributed tier-config lock while the lock quorum is still disturbed.
|
||||
/// Classifying those rejections as terminal retired the reload worker on its
|
||||
/// first attempt and pinned the peer to the previous configuration, so it
|
||||
/// served `TierNotFound` for an already-committed tier until an unrelated
|
||||
/// second admin mutation spawned a new worker.
|
||||
#[test]
|
||||
fn tier_config_reload_retries_peers_that_reject_the_apply_mid_restart() {
|
||||
for error_info in [
|
||||
"Lock acquisition timeout for resource '.rustfs.sys/config/tier-config.bin.lock' after 5s",
|
||||
"Resource '.rustfs.sys/config/tier-config.bin.lock' is already locked by node-3",
|
||||
"Internal error: release RPC failed: transport error",
|
||||
"save_config_with_opts: err: PreconditionFailed",
|
||||
"erasure read quorum",
|
||||
] {
|
||||
assert!(
|
||||
matches!(
|
||||
tier_config_reload_remote_failure(Some(error_info.to_string())),
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(_)
|
||||
),
|
||||
"a peer that rejected the apply must stay retryable so it converges: {error_info}"
|
||||
);
|
||||
}
|
||||
|
||||
// An absent error message is still a rejection, not a reason to stop.
|
||||
assert!(matches!(
|
||||
tier_config_reload_remote_failure(None),
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(_)
|
||||
));
|
||||
|
||||
// tonic surfaces a connection dropped mid-call as `Unknown`, not `Unavailable`.
|
||||
assert!(matches!(
|
||||
tier_config_reload_status_outcome(tonic::Status::unknown("transport error")),
|
||||
TierConfigReloadOutcome::TransientReconnect(_)
|
||||
));
|
||||
// An `Unknown` that is not transport-shaped stays terminal.
|
||||
assert!(matches!(
|
||||
tier_config_reload_status_outcome(tonic::Status::unknown("peer response unknown")),
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2500,22 +2271,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_decoder_fails_closed() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert_eq!(
|
||||
decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
|
||||
epoch
|
||||
);
|
||||
assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
|
||||
assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
|
||||
let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
|
||||
}
|
||||
|
||||
struct TierMutationResponseFixture<'a> {
|
||||
version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
@@ -2740,39 +2495,6 @@ mod tests {
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_app_errors_mentioning_unavailable() {
|
||||
// Regression: application error text containing "unavailable" (a
|
||||
// remote error_info payload, or a bucket named "unavailable-logs" in
|
||||
// a typed application status) must not take a healthy peer offline.
|
||||
let client = test_peer_client();
|
||||
let err = client
|
||||
.finalize_result::<()>(Err(Error::other("peer replication statistics provider is unavailable")))
|
||||
.await
|
||||
.expect_err("application error should still be returned");
|
||||
assert!(err.to_string().contains("provider is unavailable"));
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
|
||||
let err = client
|
||||
.finalize_result::<()>(Err(Error::from(tonic::Status::internal(
|
||||
"failed to reload metadata for bucket \"unavailable-logs\"",
|
||||
))))
|
||||
.await
|
||||
.expect_err("application status should still be returned");
|
||||
assert!(err.to_string().contains("unavailable-logs"));
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_marks_offline_for_typed_unavailable_status() {
|
||||
let client = test_peer_client();
|
||||
client
|
||||
.finalize_result::<()>(Err(Error::from(tonic::Status::unavailable("storage layer is not initialized"))))
|
||||
.await
|
||||
.expect_err("network error should still be returned");
|
||||
assert!(client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn peer_rest_recovery_probe_logs_keep_request_id_span_context() {
|
||||
let logs = CapturedLogs::default();
|
||||
@@ -2787,11 +2509,6 @@ mod tests {
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
// The `recovery-monitor` callsite is shared with the production
|
||||
// `mark_offline_and_spawn_recovery` path that sibling tests exercise from
|
||||
// subscriber-less threads; without this the span can be cached as
|
||||
// `Interest::never()` and silently degrade to `Span::none()`.
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
let client = test_peer_client();
|
||||
let span = tracing::info_span!("request-span", request_id = "req-peer-rest");
|
||||
|
||||
@@ -14,10 +14,8 @@
|
||||
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
|
||||
node_service_time_out_client,
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
|
||||
@@ -41,84 +39,16 @@ use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClie
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use std::sync::{
|
||||
Mutex as StdMutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
|
||||
#[cfg(test)]
|
||||
use tokio::sync::Notify;
|
||||
use tokio::{net::TcpStream, sync::RwLock, time};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
type Client = Arc<Box<dyn PeerS3Client>>;
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
pub(crate) struct DeleteBucketEmptyScanBarrier {
|
||||
arrived: AtomicBool,
|
||||
arrived_notify: Notify,
|
||||
released: AtomicBool,
|
||||
release_notify: Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl DeleteBucketEmptyScanBarrier {
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
loop {
|
||||
let notified = self.arrived_notify.notified();
|
||||
if self.arrived.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.released.store(true, Ordering::Release);
|
||||
self.release_notify.notify_waiters();
|
||||
}
|
||||
|
||||
async fn pause(&self) {
|
||||
self.arrived.store(true, Ordering::Release);
|
||||
self.arrived_notify.notify_waiters();
|
||||
loop {
|
||||
let notified = self.release_notify.notified();
|
||||
if self.released.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
|
||||
let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
|
||||
*DELETE_BUCKET_EMPTY_SCAN_BARRIER
|
||||
.lock()
|
||||
.expect("empty scan barrier lock should not be poisoned") = Some(barrier.clone());
|
||||
barrier
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_after_delete_bucket_empty_scan() {
|
||||
let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
|
||||
.lock()
|
||||
.expect("empty scan barrier lock should not be poisoned")
|
||||
.take();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.pause().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ScannerBucketListing {
|
||||
pub buckets: Vec<BucketInfo>,
|
||||
@@ -159,22 +89,6 @@ fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Erro
|
||||
reduce_write_quorum_errs(per_pool_errs, BUCKET_OP_IGNORED_ERRS, pool_write_quorum(per_pool_errs.len()))
|
||||
}
|
||||
|
||||
fn resolve_heal_bucket_mode(opts: &mut HealOpts, pool_errs: &[Option<Error>]) -> Result<()> {
|
||||
if opts.recreate {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(err) = pool_errs
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|err| **err != Error::DiskNotFound && **err != Error::VolumeNotFound)
|
||||
{
|
||||
return Err(err.clone());
|
||||
}
|
||||
opts.remove = is_all_buckets_not_found(pool_errs);
|
||||
opts.recreate = !opts.remove;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait PeerS3Client: Debug + Sync + Send + 'static {
|
||||
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
|
||||
@@ -245,7 +159,10 @@ impl S3PeerSys {
|
||||
pool_errs.push(reduce_pool_write_quorum_errs(&per_pool_errs));
|
||||
}
|
||||
|
||||
resolve_heal_bucket_mode(&mut opts, &pool_errs)?;
|
||||
if !opts.recreate {
|
||||
opts.remove = is_all_buckets_not_found(&pool_errs);
|
||||
opts.recreate = !opts.remove;
|
||||
}
|
||||
|
||||
let mut futures = Vec::new();
|
||||
let heal_bucket_results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.clients.len()]));
|
||||
@@ -719,22 +636,24 @@ impl PeerS3Client for LocalPeerS3Client {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
if opts.force_if_empty && !opts.force {
|
||||
let force = if opts.force_if_empty && !opts.force {
|
||||
for disk in local_disks.iter() {
|
||||
if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? {
|
||||
return Err(Error::VolumeNotEmpty);
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_after_delete_bucket_empty_scan().await;
|
||||
}
|
||||
true
|
||||
} else {
|
||||
opts.force
|
||||
};
|
||||
|
||||
let mut futures = Vec::with_capacity(local_disks.len());
|
||||
|
||||
for disk in local_disks.iter() {
|
||||
// `force_if_empty` is validation-only. Passing it as force would let
|
||||
// a PutObject committed after the scan be removed recursively.
|
||||
futures.push(disk.delete_volume(bucket, opts.force));
|
||||
// Non-force delete refuses a non-empty bucket (VolumeNotEmpty), which
|
||||
// the recreate loop below turns into BucketNotEmpty; only an explicit
|
||||
// force delete removes recursively (backlog#799 B1).
|
||||
futures.push(disk.delete_volume(bucket, force));
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
@@ -797,15 +716,6 @@ pub struct RemotePeerS3Client {
|
||||
}
|
||||
|
||||
impl RemotePeerS3Client {
|
||||
fn encode_delete_bucket_options(opts: &DeleteBucketOptions) -> Result<String> {
|
||||
let mut remote_opts = opts.clone();
|
||||
// Older peers promote `force_if_empty` to recursive force after their
|
||||
// metadata scan. Keep this coordinator-only hint off the wire so a
|
||||
// mixed-version delete fails closed on non-empty directory remnants.
|
||||
remote_opts.force_if_empty = false;
|
||||
serde_json::to_string(&remote_opts).map_err(Into::into)
|
||||
}
|
||||
|
||||
fn recovery_monitor_span(addr: &str) -> tracing::Span {
|
||||
tracing::info_span!(
|
||||
"recovery-monitor",
|
||||
@@ -832,7 +742,7 @@ impl RemotePeerS3Client {
|
||||
client
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
|
||||
@@ -1007,11 +917,10 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
|| async {
|
||||
let options: String = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(HealBucketRequest {
|
||||
let request = Request::new(HealBucketRequest {
|
||||
bucket: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let response = client.heal_bucket(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return if let Some(err) = response.error {
|
||||
@@ -1064,11 +973,10 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
|| async {
|
||||
let options = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MakeBucketRequest {
|
||||
let request = Request::new(MakeBucketRequest {
|
||||
name: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let response = client.make_bucket(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
@@ -1116,14 +1024,13 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let options = Self::encode_delete_bucket_options(opts)?;
|
||||
let options = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
|
||||
let mut request = Request::new(DeleteBucketRequest {
|
||||
let request = Request::new(DeleteBucketRequest {
|
||||
bucket: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let response = client.delete_bucket(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return if let Some(err) = response.error {
|
||||
@@ -1490,49 +1397,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_delete_bucket_options_fail_closed_for_legacy_peers() {
|
||||
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
|
||||
no_lock: true,
|
||||
no_recreate: true,
|
||||
force_if_empty: true,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("remote delete options should serialize");
|
||||
let legacy_opts: DeleteBucketOptions =
|
||||
serde_json::from_str(&encoded).expect("legacy peer should decode remote delete options");
|
||||
|
||||
assert!(legacy_opts.no_lock);
|
||||
assert!(legacy_opts.no_recreate);
|
||||
assert!(!legacy_opts.force);
|
||||
assert!(!legacy_opts.force_if_empty);
|
||||
|
||||
let legacy_recursive_force = if legacy_opts.force_if_empty && !legacy_opts.force {
|
||||
true
|
||||
} else {
|
||||
legacy_opts.force
|
||||
};
|
||||
assert!(
|
||||
!legacy_recursive_force,
|
||||
"legacy peer must not upgrade empty-only delete to recursive force"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_delete_bucket_options_preserve_explicit_force() {
|
||||
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
|
||||
force: true,
|
||||
force_if_empty: true,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("remote force-delete options should serialize");
|
||||
let remote_opts: DeleteBucketOptions =
|
||||
serde_json::from_str(&encoded).expect("remote peer should decode force-delete options");
|
||||
|
||||
assert!(remote_opts.force);
|
||||
assert!(!remote_opts.force_if_empty);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
|
||||
let client = test_remote_peer("http://peer-network-error:9000");
|
||||
@@ -1690,54 +1554,6 @@ mod tests {
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn local_peer_force_if_empty_preserves_unclassified_file_in_selected_pool() {
|
||||
reset_local_disk_test_state().await;
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for empty-only delete regression");
|
||||
let disks = init_test_local_disks_for_pools(
|
||||
&temp_dir,
|
||||
&[(0, 1), (1, 1)],
|
||||
"local-peer-force-if-empty-preserves-unclassified-file",
|
||||
)
|
||||
.await;
|
||||
let bucket = "empty-only-delete-bucket";
|
||||
let marker = "object/commit-marker";
|
||||
let data = bytes::Bytes::from_static(b"committed object data");
|
||||
|
||||
disks[1]
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("bucket should be created in the selected pool");
|
||||
disks[1]
|
||||
.write_all(bucket, marker, data.clone())
|
||||
.await
|
||||
.expect("unclassified committed file should be written");
|
||||
|
||||
let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
|
||||
.delete_bucket(
|
||||
bucket,
|
||||
&DeleteBucketOptions {
|
||||
force_if_empty: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("empty-only delete must not recursively remove an unclassified file");
|
||||
|
||||
assert_eq!(err, Error::VolumeNotEmpty);
|
||||
assert_eq!(
|
||||
disks[1]
|
||||
.read_all(bucket, marker)
|
||||
.await
|
||||
.expect("unclassified committed file should be preserved"),
|
||||
data
|
||||
);
|
||||
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_bucket_local_recreates_missing_bucket_volumes() {
|
||||
@@ -1802,30 +1618,6 @@ mod tests {
|
||||
assert_eq!(err, Error::VolumeExists);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_bucket_mode_fails_closed_on_incomplete_topology() {
|
||||
let mut opts = HealOpts::default();
|
||||
assert_eq!(
|
||||
resolve_heal_bucket_mode(&mut opts, &[Some(Error::ErasureWriteQuorum)]),
|
||||
Err(Error::ErasureWriteQuorum)
|
||||
);
|
||||
assert!(!opts.recreate);
|
||||
assert!(!opts.remove);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_bucket_mode_distinguishes_deleted_and_partial_buckets() {
|
||||
let mut deleted = HealOpts::default();
|
||||
resolve_heal_bucket_mode(&mut deleted, &[Some(Error::VolumeNotFound)]).unwrap();
|
||||
assert!(deleted.remove);
|
||||
assert!(!deleted.recreate);
|
||||
|
||||
let mut partial = HealOpts::default();
|
||||
resolve_heal_bucket_mode(&mut partial, &[None, Some(Error::VolumeNotFound)]).unwrap();
|
||||
assert!(!partial.remove);
|
||||
assert!(partial.recreate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_make_bucket_reduces_quorum_by_pool_participants() {
|
||||
let peer_sys = S3PeerSys {
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
|
||||
node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||
};
|
||||
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
@@ -25,7 +25,7 @@ use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
disk_store::{
|
||||
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
|
||||
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
|
||||
@@ -50,9 +50,8 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
|
||||
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
|
||||
RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest,
|
||||
WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -71,7 +70,7 @@ use tokio::{
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -101,18 +100,6 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
|
||||
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
|
||||
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
|
||||
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
|
||||
pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
|
||||
|
||||
fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.protocol_version != SNAPSHOT_LEASE_PROTOCOL_VERSION {
|
||||
return Err(Error::other("remote snapshot lease protocol is incompatible"));
|
||||
}
|
||||
SnapshotLeaseToken::from_slice(&response.token)
|
||||
}
|
||||
|
||||
/// Bind a mutating disk RPC to its canonical body: the digest lands in the request metadata, and
|
||||
/// the signing interceptor folds it (plus a replay-protected nonce) into the v2 signature scope
|
||||
@@ -1083,7 +1070,7 @@ impl RemoteDisk {
|
||||
internode_offline_bypass_reason(&self.addr).map(Error::other)
|
||||
}
|
||||
|
||||
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1096,7 +1083,7 @@ impl RemoteDisk {
|
||||
/// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
|
||||
/// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
|
||||
/// is disabled.
|
||||
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1797,81 +1784,6 @@ impl DiskAPI for RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SnapshotLeaseRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
|
||||
.map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "acquire_snapshot_lease")?;
|
||||
let response = client.acquire_snapshot_lease(request).await?.into_inner();
|
||||
snapshot_lease_token_from_response(response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SnapshotLeaseRenewRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
|
||||
.map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "renew_snapshot_lease")?;
|
||||
let response = client.renew_snapshot_lease(request).await?.into_inner();
|
||||
snapshot_lease_token_from_response(response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SnapshotLeaseReleaseRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_release_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "release_snapshot_lease")?;
|
||||
let response = client.release_snapshot_lease(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
trace!(
|
||||
@@ -3005,7 +2917,6 @@ mod tests {
|
||||
use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::io::{self as std_io, Write};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex, Mutex as StdMutex, Once};
|
||||
@@ -3019,48 +2930,6 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
// `#[serial(internode_metrics)]` marks every test that observes
|
||||
// `global_internode_metrics()`. Those counters are a process-wide singleton:
|
||||
// some of these tests snapshot a counter, run one decode, and assert on the
|
||||
// delta, while others deliberately record decode errors or call
|
||||
// `reset_internode_metrics_for_test()`. Run concurrently in one process they
|
||||
// corrupt each other's deltas — a sibling's error bumps the "no decode error"
|
||||
// assertion off zero, and a sibling's reset can drive an `after > before`
|
||||
// assertion backwards.
|
||||
//
|
||||
// The marker only takes effect under the `cargo test` fallback; nextest
|
||||
// already isolates each test in its own process, so every test there gets its
|
||||
// own copy of the counters (see `docs/testing/README.md`). Any new test that
|
||||
// reads or mutates the global internode metrics belongs in this group.
|
||||
|
||||
#[test]
|
||||
fn snapshot_lease_response_requires_current_protocol_and_valid_token() {
|
||||
let token = SnapshotLeaseToken::new();
|
||||
let response = SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
|
||||
error: None,
|
||||
};
|
||||
assert_eq!(snapshot_lease_token_from_response(response).unwrap(), token);
|
||||
|
||||
let incompatible = SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION + 1,
|
||||
error: None,
|
||||
};
|
||||
assert!(snapshot_lease_token_from_response(incompatible).is_err());
|
||||
|
||||
let malformed = SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: Bytes::from_static(b"not-a-uuid"),
|
||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
|
||||
error: None,
|
||||
};
|
||||
assert!(snapshot_lease_token_from_response(malformed).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_volumes_decode_rejects_a_malformed_entry() {
|
||||
let valid = serde_json::to_string(&VolumeInfo {
|
||||
@@ -3321,7 +3190,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_prefers_msgpack_payloads() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary");
|
||||
@@ -3344,7 +3212,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_falls_back_to_json_payloads() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let json_resp = sample_read_multiple_resp("json", b"fallback");
|
||||
@@ -3366,7 +3233,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn rename_data_response_accepts_legacy_json_without_decode_error() {
|
||||
crate::cluster::rpc::runtime_sources::reset_internode_metrics_for_test();
|
||||
let response = RenameDataResp {
|
||||
@@ -3518,7 +3384,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = ReadMultipleResponse {
|
||||
@@ -3544,7 +3409,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_reports_corrupt_json_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = ReadMultipleResponse {
|
||||
@@ -3585,7 +3449,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_decode_prefers_msgpack_payloads() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true);
|
||||
@@ -3606,7 +3469,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_rejects_invalid_success_metadata() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let mut response_item = sample_batch_read_version_resp(0, "invalid-object", true);
|
||||
@@ -3626,7 +3488,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_decode_reports_corrupt_msgpack_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = BatchReadVersionResponse {
|
||||
@@ -3653,7 +3514,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_decode_reports_corrupt_json_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = BatchReadVersionResponse {
|
||||
@@ -4443,7 +4303,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(internode_metrics)]
|
||||
async fn test_remote_disk_create_file_retries_once_on_retryable_open_write_error() {
|
||||
let transport = RetryingOpenWriteInternodeDataTransport::with_steps(vec![
|
||||
OpenWriteTestStep::Error(DiskError::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
@@ -4482,7 +4341,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(internode_metrics)]
|
||||
async fn test_remote_disk_read_file_stream_retries_once_on_retryable_open_read_error() {
|
||||
// A transient reset-by-peer on a shard read during the read-after-write window must be
|
||||
// absorbed by one re-dial rather than eroding read quorum (issue #2761).
|
||||
@@ -5332,11 +5190,6 @@ mod tests {
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
// The `recovery-monitor` span and the monitor's own log events are
|
||||
// production callsites that sibling tests exercise from subscriber-less
|
||||
// threads; without this they can be cached as `Interest::never()` and go
|
||||
// silently missing here.
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
let endpoint = Endpoint {
|
||||
url: url::Url::parse("http://127.0.0.1:59996/data").expect("endpoint URL should parse"),
|
||||
@@ -5391,11 +5244,6 @@ mod tests {
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
// The `recovery-monitor` span and the monitor's own log events are
|
||||
// production callsites that sibling tests exercise from subscriber-less
|
||||
// threads; without this they can be cached as `Interest::never()` and go
|
||||
// silently missing here.
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
let addr = "http://127.0.0.1:59997".to_string();
|
||||
let endpoint = Endpoint {
|
||||
|
||||
@@ -12,10 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use rustfs_lock::{
|
||||
@@ -31,6 +28,7 @@ use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// Remote lock client implementation
|
||||
@@ -79,7 +77,7 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
// P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
|
||||
// offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
|
||||
// change quorum; the self-healing re-probe keeps the peer recoverable.
|
||||
@@ -315,11 +313,10 @@ impl LockClient for RemoteClient {
|
||||
info!("remote acquire_exclusive for {}", request.resource);
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = match self.execute_rpc("lock", &resource_summary, client.lock(req)).await {
|
||||
Ok(resp) => resp.into_inner(),
|
||||
@@ -350,7 +347,7 @@ impl LockClient for RemoteClient {
|
||||
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(requests);
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
let req = Request::new(BatchGenerallyLockRequest {
|
||||
args: requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
@@ -358,7 +355,6 @@ impl LockClient for RemoteClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc("lock_batch", &resource_summary, client.lock_batch(req))
|
||||
@@ -399,8 +395,7 @@ impl LockClient for RemoteClient {
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = unlock_request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
let req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
let resp = self
|
||||
.execute_rpc("release", &resource_summary, client.un_lock(req))
|
||||
.await?
|
||||
@@ -419,7 +414,7 @@ impl LockClient for RemoteClient {
|
||||
let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(&unlock_requests);
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
let req = Request::new(BatchGenerallyLockRequest {
|
||||
args: unlock_requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
@@ -427,7 +422,6 @@ impl LockClient for RemoteClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = self
|
||||
.execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
|
||||
@@ -446,11 +440,10 @@ impl LockClient for RemoteClient {
|
||||
let refresh_request = Self::create_unlock_request(lock_id);
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = refresh_request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&refresh_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("refresh", &resource_summary, client.refresh(req))
|
||||
.await?
|
||||
@@ -466,11 +459,10 @@ impl LockClient for RemoteClient {
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = force_request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&force_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
|
||||
.await?
|
||||
@@ -491,11 +483,10 @@ impl LockClient for RemoteClient {
|
||||
let mut client = self.get_client().await?;
|
||||
|
||||
// Try to acquire a very short-lived lock to test availability
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
// Try exclusive lock first with very short timeout
|
||||
let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
|
||||
@@ -506,11 +497,10 @@ impl LockClient for RemoteClient {
|
||||
if resp.success {
|
||||
// If we successfully acquired the lock, the resource was free.
|
||||
// Immediately release it on a best-effort basis.
|
||||
let mut release_req = Request::new(GenerallyLockRequest {
|
||||
let release_req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut release_req)?;
|
||||
let _ = self
|
||||
.execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
|
||||
.await;
|
||||
|
||||
@@ -2543,7 +2543,6 @@ mod tests {
|
||||
data_dir: None,
|
||||
delete_marker: false,
|
||||
transitioned_object: Default::default(),
|
||||
transition_version_state: Default::default(),
|
||||
restore_ongoing: false,
|
||||
restore_expires: None,
|
||||
user_tags: Arc::new(String::new()),
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::{
|
||||
CheckPartsResp, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint,
|
||||
Error, FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result,
|
||||
SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error,
|
||||
FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
health_state::{
|
||||
RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
|
||||
get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
|
||||
@@ -1349,38 +1349,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.acquire_snapshot_lease(volume, path).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.release_snapshot_lease(volume, path, token).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.renew_snapshot_lease(volume, path, token).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.delete_data_dir(volume, path, opts).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.write_metadata(org_volume, volume, path, fi).await },
|
||||
|
||||
@@ -337,54 +337,9 @@ impl From<DiskError> for std::io::Error {
|
||||
}
|
||||
}
|
||||
|
||||
/// The single in-band representation of a failed internode RPC: it carries the
|
||||
/// typed `tonic::Status` so failure classifiers can read the gRPC code instead
|
||||
/// of substring-matching the rendered message (see `is_network_like_status`).
|
||||
/// Both `DiskError` and `StorageError` wrap statuses in this type, so one
|
||||
/// downcast recovers the status regardless of which error the status was
|
||||
/// converted into first.
|
||||
pub(crate) struct RpcStatusError(tonic::Status);
|
||||
|
||||
impl RpcStatusError {
|
||||
pub(crate) fn status(&self) -> &tonic::Status {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<tonic::Status> for RpcStatusError {
|
||||
fn from(status: tonic::Status) -> Self {
|
||||
Self(status)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for RpcStatusError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
std::fmt::Display::fmt(&self.0, f)
|
||||
}
|
||||
}
|
||||
|
||||
/// `tonic::Status`'s own `Debug` prints its `MetadataMap`, i.e. every response
|
||||
/// header and trailer the peer sent. Those are remote-controlled and can carry
|
||||
/// credentials injected by a proxy in front of the peer, so keep them out of
|
||||
/// anything that reaches a log.
|
||||
impl std::fmt::Debug for RpcStatusError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RpcStatusError")
|
||||
.field("code", &self.0.code())
|
||||
.field("message", &self.0.message())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl StdError for RpcStatusError {
|
||||
fn source(&self) -> Option<&(dyn StdError + 'static)> {
|
||||
Some(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<tonic::Status> for DiskError {
|
||||
fn from(e: tonic::Status) -> Self {
|
||||
DiskError::Io(io::Error::other(RpcStatusError(e)))
|
||||
DiskError::other(e.message().to_string())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+51
-1537
File diff suppressed because it is too large
Load Diff
@@ -72,39 +72,6 @@ pub type DiskStore = Arc<Disk>;
|
||||
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
|
||||
pub type FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SnapshotLeaseToken(Uuid);
|
||||
|
||||
impl SnapshotLeaseToken {
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::new_v4())
|
||||
}
|
||||
|
||||
pub fn from_slice(bytes: &[u8]) -> Result<Self> {
|
||||
let uuid = Uuid::from_slice(bytes).map_err(|_| Error::other("invalid snapshot lease token"))?;
|
||||
if uuid.is_nil() {
|
||||
return Err(Error::other("invalid snapshot lease token"));
|
||||
}
|
||||
Ok(Self(uuid))
|
||||
}
|
||||
|
||||
pub fn as_bytes(&self) -> &[u8; 16] {
|
||||
self.0.as_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SnapshotLeaseToken {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum DataDirDeleteStatus {
|
||||
Deleted,
|
||||
Deferred,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PartTransactionAction {
|
||||
Commit,
|
||||
@@ -282,34 +249,6 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.acquire_snapshot_lease(volume, path).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.acquire_snapshot_lease(volume, path).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.release_snapshot_lease(volume, path, token).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.release_snapshot_lease(volume, path, token).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.renew_snapshot_lease(volume, path, token).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.renew_snapshot_lease(volume, path, token).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.delete_data_dir(volume, path, opts).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.delete_data_dir(volume, path, opts).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
match self {
|
||||
@@ -707,19 +646,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
) -> Result<()>;
|
||||
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>>;
|
||||
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()>;
|
||||
async fn acquire_snapshot_lease(&self, _volume: &str, _path: &str) -> Result<SnapshotLeaseToken> {
|
||||
Err(Error::other("snapshot leases are not supported by this disk"))
|
||||
}
|
||||
async fn release_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<()> {
|
||||
Err(Error::other("snapshot leases are not supported by this disk"))
|
||||
}
|
||||
async fn renew_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
Err(Error::other("snapshot leases are not supported by this disk"))
|
||||
}
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
self.delete(volume, path, opts).await?;
|
||||
Ok(DataDirDeleteStatus::Deleted)
|
||||
}
|
||||
async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
|
||||
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()>;
|
||||
async fn read_version(
|
||||
|
||||
@@ -25,7 +25,7 @@ use std::{
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit};
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
|
||||
use tracing::warn;
|
||||
|
||||
/// Check path length according to OS limits.
|
||||
@@ -123,8 +123,6 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
|
||||
|
||||
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
|
||||
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
|
||||
let cpu_scaled = cpu_count
|
||||
@@ -159,24 +157,6 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
|
||||
limiter
|
||||
}
|
||||
|
||||
/// Serialize a bucket's local metadata commits with physical bucket removal.
|
||||
///
|
||||
/// The key includes the canonical disk root, so independently reconnected
|
||||
/// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
|
||||
/// disconnected disks do not keep the registry alive.
|
||||
pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
|
||||
let key = root.join(volume);
|
||||
let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
|
||||
locks.retain(|_, lock| lock.strong_count() > 0);
|
||||
if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
|
||||
return lock;
|
||||
}
|
||||
|
||||
let lock = Arc::new(RwLock::new(()));
|
||||
locks.insert(key, Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
|
||||
/// Always acquire the per-disk permit before the process-wide permit. Keeping
|
||||
/// this order uniform prevents one slow disk from reserving global capacity
|
||||
/// while it waits for its own concurrency slot.
|
||||
@@ -592,14 +572,15 @@ async fn reliable_rename_inner(
|
||||
base_dir: impl AsRef<Path>,
|
||||
warn_on_missing_source: bool,
|
||||
) -> io::Result<()> {
|
||||
let parent_guard = match dst_file_path.as_ref().parent() {
|
||||
Some(parent) => Some(mkdir_all_below_existing_base(parent, base_dir.as_ref()).await?),
|
||||
None => None,
|
||||
};
|
||||
if let Some(parent) = dst_file_path.as_ref().parent()
|
||||
&& !file_exists(parent)
|
||||
{
|
||||
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
|
||||
}
|
||||
|
||||
let mut i = 0;
|
||||
loop {
|
||||
if let Err(e) = rename_into_existing_parent(src_file_path.as_ref(), dst_file_path.as_ref(), parent_guard.as_ref()) {
|
||||
if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
|
||||
if should_retry_rename(&e, i) {
|
||||
i += 1;
|
||||
continue;
|
||||
@@ -616,159 +597,6 @@ async fn reliable_rename_inner(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn rename_into_existing_parent(
|
||||
src_file_path: &Path,
|
||||
dst_file_path: &Path,
|
||||
parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
) -> io::Result<()> {
|
||||
use rustix::fs::{Mode, OFlags, open, renameat};
|
||||
|
||||
let Some(parent_guard) = parent_guard else {
|
||||
return super::fs::rename_std(src_file_path, dst_file_path);
|
||||
};
|
||||
let src_parent = src_file_path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
|
||||
let src_name = src_file_path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a file name"))?;
|
||||
let dst_name = dst_file_path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?;
|
||||
let src_parent = open(
|
||||
src_parent,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
let dst_parent = parent_guard
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?;
|
||||
|
||||
renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from)
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn rename_into_existing_parent(
|
||||
src_file_path: &Path,
|
||||
dst_file_path: &Path,
|
||||
_parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
) -> io::Result<()> {
|
||||
super::fs::rename_std(src_file_path, dst_file_path)
|
||||
}
|
||||
|
||||
async fn mkdir_all_below_existing_base(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
let dir_path = dir_path.to_path_buf();
|
||||
let base_dir = base_dir.to_path_buf();
|
||||
|
||||
tokio::task::spawn_blocking(move || mkdir_all_below_existing_base_std(&dir_path, &base_dir)).await?
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) type ExistingBaseDirectoryGuard = Vec<winapi_util::Handle>;
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) type ExistingBaseDirectoryGuard = Vec<std::os::fd::OwnedFd>;
|
||||
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
pub(crate) type ExistingBaseDirectoryGuard = ();
|
||||
|
||||
#[cfg(windows)]
|
||||
fn lock_windows_directory(path: &Path) -> io::Result<winapi_util::Handle> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
|
||||
const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
|
||||
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
|
||||
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
|
||||
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
|
||||
const FILE_SHARE_READ: u32 = 0x1;
|
||||
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(FILE_SHARE_READ)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(path)?;
|
||||
let handle = winapi_util::Handle::from_file(file);
|
||||
let info = winapi_util::file::information(&handle)?;
|
||||
if info.file_attributes() & FILE_ATTRIBUTE_DIRECTORY == 0
|
||||
|| info.file_attributes() & u64::from(FILE_ATTRIBUTE_REPARSE_POINT) != 0
|
||||
{
|
||||
return Err(io::Error::from(io::ErrorKind::NotADirectory));
|
||||
}
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
pub(crate) fn mkdir_all_below_existing_base_std(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
let relative = dir_path
|
||||
.strip_prefix(base_dir)
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?;
|
||||
for component in relative.components() {
|
||||
if !matches!(component, Component::Normal(_) | Component::CurDir) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"rename destination contains an invalid path component",
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
|
||||
use rustix::io::Errno;
|
||||
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
|
||||
let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
|
||||
|
||||
for component in relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
continue;
|
||||
};
|
||||
let parent = parents
|
||||
.last()
|
||||
.expect("base directory guard should contain the base directory");
|
||||
match mkdirat(parent, component, mode) {
|
||||
Ok(()) => {}
|
||||
Err(Errno::EXIST) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
|
||||
}
|
||||
|
||||
Ok(parents)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let mut handles = vec![lock_windows_directory(base_dir)?];
|
||||
let mut current = base_dir.to_path_buf();
|
||||
for component in relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
continue;
|
||||
};
|
||||
current.push(component);
|
||||
match std::fs::create_dir(¤t) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
handles.push(lock_windows_directory(¤t)?);
|
||||
}
|
||||
|
||||
Ok(handles)
|
||||
}
|
||||
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
{
|
||||
let _ = relative;
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Unsupported,
|
||||
"safe recursive directory creation is unavailable on this platform",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
|
||||
warn!(
|
||||
"reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
|
||||
@@ -899,20 +727,6 @@ mod tests {
|
||||
Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_volume_mutation_lock_is_shared_per_root_and_volume() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first = disk_volume_mutation_lock(temp_dir.path(), "bucket");
|
||||
let second = disk_volume_mutation_lock(temp_dir.path(), "bucket");
|
||||
let other = disk_volume_mutation_lock(temp_dir.path(), "other-bucket");
|
||||
|
||||
assert!(Arc::ptr_eq(&first, &second), "reconnected disks must share a bucket mutation lock");
|
||||
assert!(!Arc::ptr_eq(&first, &other), "different buckets must not serialize each other");
|
||||
|
||||
let _write_guard = first.write().await;
|
||||
assert!(second.try_read().is_err(), "a bucket delete lock must exclude local commits");
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
@@ -957,26 +771,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Holds a `warn_capture()` capture alive: the thread-local subscriber, plus
|
||||
/// the pin that keeps tracing's process-global callsite-interest cache from
|
||||
/// being decided by some other test's thread.
|
||||
struct WarnCaptureGuard {
|
||||
_subscriber: tracing::subscriber::DefaultGuard,
|
||||
_callsite_pin: tracing::Dispatch,
|
||||
}
|
||||
|
||||
/// Capture WARN-level output on the current thread; tokio tests here run on
|
||||
/// the current-thread runtime, so the guard covers the whole test body.
|
||||
///
|
||||
/// The callsite pin matters because `warn_reliable_rename_failure` is a
|
||||
/// single production callsite shared with tests that call `rename_all`
|
||||
/// *without* installing a subscriber — `rename_all_missing_source_returns_file_not_found`
|
||||
/// is one. Whichever thread reaches it first fixes its `Interest`
|
||||
/// process-wide, so without the pin that sibling can cache
|
||||
/// `Interest::never()` and the WARN never fires here at all, leaving the
|
||||
/// "must keep the WARN" assertions staring at empty output. See
|
||||
/// [`crate::test_tracing::pin_callsite_interest_for_test`].
|
||||
fn warn_capture() -> (CapturedLogs, WarnCaptureGuard) {
|
||||
fn warn_capture() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
@@ -984,10 +781,7 @@ mod tests {
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let guard = WarnCaptureGuard {
|
||||
_subscriber: tracing::subscriber::set_default(subscriber),
|
||||
_callsite_pin: crate::test_tracing::pin_callsite_interest_for_test(),
|
||||
};
|
||||
let guard = tracing::subscriber::set_default(subscriber);
|
||||
(logs, guard)
|
||||
}
|
||||
|
||||
@@ -1174,112 +968,6 @@ mod tests {
|
||||
assert_eq!(std::fs::read(dst.join("nested").join("part.1")).expect("read moved part"), b"payload");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_all_does_not_recreate_missing_base_directory() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
let src = temp_dir.path().join("staged-object");
|
||||
std::fs::write(&src, b"payload").expect("write staged object");
|
||||
let dst = base.join("object").join("xl.meta");
|
||||
std::fs::remove_dir(&base).expect("delete destination base before commit");
|
||||
|
||||
let err = rename_all(&src, &dst, &base)
|
||||
.await
|
||||
.expect_err("rename must not recreate a deleted destination base");
|
||||
|
||||
assert!(matches!(err, DiskError::FileNotFound));
|
||||
assert!(src.exists(), "failed commit must preserve the staged source");
|
||||
assert!(!base.exists(), "failed commit must not recreate the deleted bucket");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_all_rejects_a_replaced_base_with_an_existing_parent() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
std::fs::create_dir_all(outside.join("object")).expect("create outside destination parent");
|
||||
let src = temp_dir.path().join("staged-object");
|
||||
std::fs::write(&src, b"payload").expect("write staged object");
|
||||
let dst = base.join("object").join("xl.meta");
|
||||
|
||||
std::fs::remove_dir(&base).expect("remove destination base before replacement");
|
||||
symlink(&outside, &base).expect("replace destination base with a symlink");
|
||||
|
||||
rename_all(&src, &dst, &base)
|
||||
.await
|
||||
.expect_err("rename must reject an existing destination parent below a replaced base");
|
||||
|
||||
assert!(src.exists(), "rejected rename must preserve the staged source");
|
||||
assert!(
|
||||
!outside.join("object/xl.meta").exists(),
|
||||
"rename must not publish through the replacement symlink"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_parent_guard_blocks_base_and_intermediate_replacement() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
let parent = base.join("object").join("nested");
|
||||
let guard = mkdir_all_below_existing_base_std(&parent, &base).expect("create and lock destination parents");
|
||||
|
||||
std::fs::rename(&base, temp_dir.path().join("replacement-base"))
|
||||
.expect_err("the locked base must not be replaceable before commit");
|
||||
std::fs::rename(base.join("object"), base.join("replacement-object"))
|
||||
.expect_err("a locked intermediate directory must not be replaceable before commit");
|
||||
|
||||
drop(guard);
|
||||
std::fs::rename(base.join("object"), base.join("replacement-object"))
|
||||
.expect("replacement should succeed after the commit guard is released");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_parent_creation_rejects_symlinked_base() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
std::fs::create_dir(&outside).expect("create outside directory");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
symlink(&outside, &base).expect("create symlinked base");
|
||||
|
||||
mkdir_all_below_existing_base(&base.join("object"), &base)
|
||||
.await
|
||||
.expect_err("symlinked base must be rejected");
|
||||
|
||||
assert!(!outside.join("object").exists(), "parent creation must remain confined to the base");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_parent_creation_rejects_symlink_below_base() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
std::fs::create_dir(&outside).expect("create outside directory");
|
||||
symlink(&outside, base.join("linked")).expect("create symlink below base");
|
||||
|
||||
mkdir_all_below_existing_base(&base.join("linked/object"), &base)
|
||||
.await
|
||||
.expect_err("symlink below base must be rejected");
|
||||
|
||||
assert!(
|
||||
!outside.join("object").exists(),
|
||||
"parent creation must not follow a symlink outside the base"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fsync_dir_succeeds_on_directory() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
|
||||
@@ -818,11 +818,7 @@ impl From<s3s::xml::DeError> for Error {
|
||||
|
||||
impl From<tonic::Status> for Error {
|
||||
fn from(e: tonic::Status) -> Self {
|
||||
// Keep the typed status as the io::Error payload instead of a
|
||||
// flattened string so RPC failure classifiers can read the gRPC code
|
||||
// via downcast (see `PeerRestClient::is_network_like_error`).
|
||||
// `RpcStatusError` renders exactly as `e.to_string()` did.
|
||||
Error::other(crate::disk::error::RpcStatusError::from(e))
|
||||
Error::other(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -93,6 +93,3 @@ pub(crate) mod ecstore_validation_blackbox;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_metrics;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_tracing;
|
||||
|
||||
@@ -180,7 +180,6 @@ pub struct ObjectInfo {
|
||||
pub data_dir: Option<Uuid>,
|
||||
pub delete_marker: bool,
|
||||
pub transitioned_object: TransitionedObject,
|
||||
pub transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
pub restore_ongoing: bool,
|
||||
pub restore_expires: Option<OffsetDateTime>,
|
||||
pub user_tags: Arc<String>,
|
||||
@@ -221,7 +220,6 @@ impl Clone for ObjectInfo {
|
||||
data_dir: self.data_dir,
|
||||
delete_marker: self.delete_marker,
|
||||
transitioned_object: self.transitioned_object.clone(),
|
||||
transition_version_state: self.transition_version_state,
|
||||
restore_ongoing: self.restore_ongoing,
|
||||
restore_expires: self.restore_expires,
|
||||
user_tags: self.user_tags.clone(),
|
||||
@@ -466,11 +464,11 @@ impl ObjectInfo {
|
||||
|
||||
let transitioned_object = TransitionedObject {
|
||||
name: fi.transitioned_objname.clone(),
|
||||
version_id: fi
|
||||
.transition_version
|
||||
.clone()
|
||||
.or_else(|| fi.transition_version_id.map(|version_id| version_id.to_string()))
|
||||
.unwrap_or_default(),
|
||||
version_id: if let Some(transition_version_id) = fi.transition_version_id {
|
||||
transition_version_id.to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
status: fi.transition_status.clone(),
|
||||
free_version: fi.tier_free_version(),
|
||||
tier: fi.transition_tier.clone(),
|
||||
@@ -539,7 +537,6 @@ impl ObjectInfo {
|
||||
inlined,
|
||||
user_defined: Arc::new(metadata),
|
||||
transitioned_object,
|
||||
transition_version_state: fi.transition_version_state,
|
||||
checksum: fi.checksum.clone(),
|
||||
storage_class,
|
||||
restore_ongoing,
|
||||
|
||||
@@ -207,6 +207,10 @@ pub(crate) async fn ensure_boot_time() {
|
||||
GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_init_time() -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
rustfs_common::get_global_init_time().await
|
||||
}
|
||||
|
||||
pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option<u64> {
|
||||
if is_erasure_sd().await {
|
||||
None
|
||||
|
||||
@@ -71,7 +71,6 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
|
||||
MadminScannerMetrics {
|
||||
collected_at: metrics.collected_at,
|
||||
current_cycle: metrics.current_cycle,
|
||||
current_cycle_active: Some(metrics.current_cycle_active),
|
||||
current_started: metrics.current_started,
|
||||
cycles_completed_at: metrics.cycles_completed_at,
|
||||
ongoing_buckets: metrics.ongoing_buckets,
|
||||
@@ -399,7 +398,10 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
|
||||
|
||||
if types.contains(&MetricType::SCANNER) {
|
||||
debug!("start get scanner metrics");
|
||||
let metrics = global_metrics().report().await;
|
||||
let mut metrics = global_metrics().report().await;
|
||||
if let Some(init_time) = runtime_sources::scanner_init_time().await {
|
||||
metrics.current_started = init_time;
|
||||
}
|
||||
real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics));
|
||||
}
|
||||
|
||||
@@ -538,9 +540,7 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use rustfs_common::metrics::CurrentCycle;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
@@ -588,10 +588,7 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_partial_source_status() {
|
||||
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
|
||||
current_cycle_active: true,
|
||||
current_started,
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
|
||||
@@ -601,8 +598,6 @@ mod test {
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
let usage = scanner
|
||||
@@ -613,39 +608,6 @@ mod test {
|
||||
assert_eq!(usage.cycles, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
|
||||
let previous_init_time = *rustfs_common::globals::GLOBAL_INIT_TIME.read().await;
|
||||
let previous_cycle = global_metrics().get_cycle().await;
|
||||
let init_time = Utc::now() - chrono::Duration::hours(1);
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
||||
let cycle = CurrentCycle {
|
||||
current: 0,
|
||||
next: 1,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_start = global_metrics().start_scan_cycle_work_with_cycle(cycle).await;
|
||||
|
||||
let realtime = collect_local_metrics(MetricType::SCANNER, &CollectMetricsOpts::default()).await;
|
||||
|
||||
global_metrics()
|
||||
.finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
|
||||
.await;
|
||||
global_metrics().set_cycle(previous_cycle).await;
|
||||
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
|
||||
|
||||
let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
|
||||
let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
|
||||
let mut aggregated = RealtimeMetrics::default();
|
||||
aggregated.merge(decoded);
|
||||
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, cycle_started);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_pacing_pressure() {
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
|
||||
|
||||
@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
|
||||
use rustfs_madmin::net::NetInfo;
|
||||
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
|
||||
use rustfs_utils::XHost;
|
||||
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
|
||||
use std::collections::{HashMap, hash_map::DefaultHasher};
|
||||
use std::future::Future;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::time::{sleep, timeout};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
@@ -47,9 +47,6 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
|
||||
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
|
||||
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Cached result from the last successful admin call to a peer.
|
||||
struct PeerAdminCache {
|
||||
@@ -94,180 +91,6 @@ lazy_static! {
|
||||
pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: String,
|
||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||
expires_at: Instant,
|
||||
}
|
||||
|
||||
impl RemoteVersionStateFleetProof {
|
||||
fn token(&self) -> RemoteVersionStateFleetProofToken {
|
||||
RemoteVersionStateFleetProofToken {
|
||||
topology_fingerprint: self.topology_fingerprint.clone(),
|
||||
peer_epochs: self.peer_epochs.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct RemoteVersionStateFleetProofToken {
|
||||
topology_fingerprint: String,
|
||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RemoteVersionStateFleetProofState {
|
||||
proof: Option<RemoteVersionStateFleetProof>,
|
||||
topology_conflict: bool,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
|
||||
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
|
||||
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
|
||||
}
|
||||
|
||||
fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
|
||||
replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
|
||||
}
|
||||
|
||||
fn replace_remote_version_state_fleet_proof_in(
|
||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
||||
proof: Option<RemoteVersionStateFleetProof>,
|
||||
) {
|
||||
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
|
||||
}
|
||||
|
||||
fn publish_remote_version_state_probe_result(
|
||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
||||
topology_fingerprint: &str,
|
||||
result: Result<BTreeMap<String, Uuid>>,
|
||||
observed_at: Instant,
|
||||
) -> Option<Error> {
|
||||
match result {
|
||||
Ok(peer_epochs) => {
|
||||
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let peer_epochs = state
|
||||
.proof
|
||||
.as_ref()
|
||||
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
|
||||
.map(|proof| Arc::clone(&proof.peer_epochs))
|
||||
.unwrap_or_else(|| Arc::new(peer_epochs));
|
||||
state.proof = Some(RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: topology_fingerprint.to_string(),
|
||||
peer_epochs,
|
||||
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
|
||||
});
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
replace_remote_version_state_fleet_proof_in(slot, None);
|
||||
Some(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
|
||||
}
|
||||
|
||||
fn acquire_remote_version_state_fleet_proof_from(
|
||||
state: &RemoteVersionStateFleetProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<RemoteVersionStateFleetProofToken> {
|
||||
if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
|
||||
return None;
|
||||
}
|
||||
state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
|
||||
}
|
||||
|
||||
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
let state = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.topology_conflict {
|
||||
return false;
|
||||
}
|
||||
state.proof.as_ref().is_some_and(|current| {
|
||||
current.topology_fingerprint == *expected_topology
|
||||
&& current.topology_fingerprint == proof.topology_fingerprint
|
||||
&& Arc::ptr_eq(¤t.peer_epochs, &proof.peer_epochs)
|
||||
&& Instant::now() < current.expires_at
|
||||
})
|
||||
}
|
||||
|
||||
fn remote_version_state_fleet_proof_valid_at(
|
||||
proof: Option<&RemoteVersionStateFleetProof>,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
|
||||
}
|
||||
|
||||
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
|
||||
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
|
||||
return Err(Error::other("remote version state capability peer identity is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
|
||||
let mut state = remote_version_state_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.topology_conflict = true;
|
||||
state.proof = None;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let result = match get_global_notification_sys() {
|
||||
Some(notification_sys) => {
|
||||
match timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
|
||||
}
|
||||
}
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
};
|
||||
let topology_conflict = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.topology_conflict;
|
||||
if topology_conflict {
|
||||
replace_remote_version_state_fleet_proof(None);
|
||||
} else if let Some(err) = publish_remote_version_state_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
result,
|
||||
Instant::now(),
|
||||
) {
|
||||
debug!(error = %err, "remote version state fleet capability probe failed closed");
|
||||
}
|
||||
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
|
||||
let _ = GLOBAL_NOTIFICATION_SYS
|
||||
.set(Arc::new(NotificationSys::new(eps).await))
|
||||
@@ -292,17 +115,7 @@ pub struct NotificationSys {
|
||||
|
||||
impl NotificationSys {
|
||||
pub async fn new(eps: EndpointServerPools) -> Self {
|
||||
let expected_remote_hosts = eps
|
||||
.peer_grid_host_slots_sorted()
|
||||
.into_iter()
|
||||
.filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
|
||||
.collect::<Vec<_>>();
|
||||
let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
|
||||
let peer_topology_hosts = if peer_topology_hosts.is_empty() {
|
||||
expected_remote_hosts
|
||||
} else {
|
||||
peer_topology_hosts
|
||||
};
|
||||
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
|
||||
Self {
|
||||
peer_clients,
|
||||
@@ -312,24 +125,6 @@ impl NotificationSys {
|
||||
tier_config_reload_workers: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("remote version state capability fleet membership is incomplete"));
|
||||
}
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
|
||||
client.probe_remote_version_state(topology_fingerprint.to_string()).await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NotificationPeerErr {
|
||||
@@ -1549,11 +1344,8 @@ async fn run_tier_config_reload_worker<F, Fut>(
|
||||
}
|
||||
TierConfigReloadFinish::Pending => retry_attempt = 0,
|
||||
},
|
||||
TierConfigReloadOutcome::Terminal(err) => match sys.finish_tier_config_reload_worker(&host) {
|
||||
TierConfigReloadOutcome::Terminal(_) => match sys.finish_tier_config_reload_worker(&host) {
|
||||
TierConfigReloadFinish::Completed => {
|
||||
// This peer keeps the previous tier configuration for good, so record
|
||||
// why. Dropping the error here hides the only evidence of a divergent
|
||||
// node behind an outcome label that cannot be acted on.
|
||||
warn!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -1561,7 +1353,6 @@ async fn run_tier_config_reload_worker<F, Fut>(
|
||||
action = "reload_transition_tier_config",
|
||||
host,
|
||||
outcome = "terminal",
|
||||
error = ?err,
|
||||
"tier configuration reload stopped after a terminal outcome"
|
||||
);
|
||||
return;
|
||||
@@ -2095,183 +1886,6 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
|
||||
let now = Instant::now();
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(peer_epochs),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
|
||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
|
||||
assert!(peer_epochs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
|
||||
let now = Instant::now();
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
|
||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
|
||||
let now = Instant::now();
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
let captured = proof.token();
|
||||
let restarted = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: proof.topology_fingerprint.clone(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: proof.expires_at,
|
||||
};
|
||||
|
||||
assert!(captured != restarted.token());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
|
||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
||||
let now = Instant::now();
|
||||
let epoch = Uuid::new_v4();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
|
||||
let original = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("successful probe should publish proof")
|
||||
.token();
|
||||
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
|
||||
);
|
||||
let renewed = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("renewal should retain proof")
|
||||
.token();
|
||||
assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
|
||||
|
||||
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
|
||||
.is_none()
|
||||
);
|
||||
let replaced = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("restarted peer should publish a new proof")
|
||||
.token();
|
||||
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
||||
let now = Instant::now();
|
||||
let mut state = RemoteVersionStateFleetProofState {
|
||||
proof: Some(RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
}),
|
||||
topology_conflict: false,
|
||||
};
|
||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
|
||||
|
||||
state.topology_conflict = true;
|
||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
|
||||
.expect("first member should be admitted");
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
|
||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
||||
let now = Instant::now();
|
||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
|
||||
);
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_none());
|
||||
|
||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
|
||||
let err = notification_sys
|
||||
.probe_remote_version_state_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("an unreachable configured member must fail the fleet proof");
|
||||
assert!(err.to_string().contains("unreachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
|
||||
let err = notification_sys
|
||||
.probe_remote_version_state_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("a missing configured member slot must fail the fleet proof");
|
||||
assert!(err.to_string().contains("incomplete"));
|
||||
}
|
||||
|
||||
fn build_props(endpoint: &str) -> ServerProperties {
|
||||
ServerProperties {
|
||||
endpoint: endpoint.to_string(),
|
||||
|
||||
@@ -931,10 +931,7 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
|
||||
status: fi.transition_status.clone(),
|
||||
tier: fi.transition_tier.clone(),
|
||||
remote_object: fi.transitioned_objname.clone(),
|
||||
remote_version_id: fi
|
||||
.transition_version
|
||||
.clone()
|
||||
.or_else(|| fi.transition_version_id.map(|id| id.to_string())),
|
||||
remote_version_id: fi.transition_version_id.map(|id| id.to_string()),
|
||||
free_version_count,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -236,7 +236,6 @@ struct TierPublishTransition {
|
||||
|
||||
struct PreparedTierDriver {
|
||||
tier_name: String,
|
||||
tier_config: TierConfig,
|
||||
config_fingerprint: TierDriverFingerprint,
|
||||
backend_identity: TierDestinationId,
|
||||
exact_get_delete: bool,
|
||||
@@ -1343,7 +1342,6 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
|
||||
|
||||
struct TierDriverGeneration {
|
||||
tier_name: Arc<str>,
|
||||
tier_config: TierConfig,
|
||||
generation: DriverRevision,
|
||||
// Process-local only: this may reflect credential changes and must never be persisted or logged.
|
||||
config_fingerprint: TierDriverFingerprint,
|
||||
@@ -1351,9 +1349,6 @@ struct TierDriverGeneration {
|
||||
backend_identity: TierDestinationId,
|
||||
exact_get_delete: bool,
|
||||
driver: SharedWarmBackend,
|
||||
reconciler: tokio::sync::OnceCell<
|
||||
Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
|
||||
>,
|
||||
accepting: AtomicBool,
|
||||
active_leases: AtomicUsize,
|
||||
drained: tokio::sync::Notify,
|
||||
@@ -1478,35 +1473,6 @@ impl TierOperationLease {
|
||||
self.inner.backend_identity
|
||||
}
|
||||
|
||||
pub(crate) async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
transaction_id: uuid::Uuid,
|
||||
) -> io::Result<TransitionCandidateProbe> {
|
||||
let Some(reconciler) = self
|
||||
.inner
|
||||
.reconciler
|
||||
.get_or_try_init(|| async {
|
||||
crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
|
||||
.await
|
||||
.map(|reconciler| reconciler.map(Arc::from))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| io::Error::other(err.message))?
|
||||
else {
|
||||
return self.inner.driver.probe_transition_candidate(object).await;
|
||||
};
|
||||
reconciler
|
||||
.probe_transition_candidate_for(
|
||||
object,
|
||||
crate::services::tier::warm_backend::TransitionCandidateIdentity {
|
||||
transaction_id,
|
||||
destination_id: self.backend_identity(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
|
||||
self.inner.driver.validate_remote_version_id(remote_version_id)?;
|
||||
if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
|
||||
@@ -2867,7 +2833,6 @@ impl TierConfigMgr {
|
||||
let exact_get_delete = tier_exact_get_delete(config);
|
||||
Some(PreparedTierDriver {
|
||||
tier_name: tier_name.to_string(),
|
||||
tier_config: config.clone(),
|
||||
config_fingerprint,
|
||||
backend_identity,
|
||||
exact_get_delete,
|
||||
@@ -2896,13 +2861,11 @@ impl TierConfigMgr {
|
||||
})?;
|
||||
let entry = Arc::new(TierDriverGeneration {
|
||||
tier_name: Arc::from(prepared.tier_name.as_str()),
|
||||
tier_config: prepared.tier_config.clone(),
|
||||
generation,
|
||||
config_fingerprint: prepared.config_fingerprint,
|
||||
backend_identity: prepared.backend_identity,
|
||||
exact_get_delete: prepared.exact_get_delete,
|
||||
driver: prepared.driver.clone(),
|
||||
reconciler: tokio::sync::OnceCell::new(),
|
||||
accepting: AtomicBool::new(true),
|
||||
active_leases: AtomicUsize::new(0),
|
||||
drained: tokio::sync::Notify::new(),
|
||||
@@ -3646,13 +3609,11 @@ impl TierConfigMgr {
|
||||
let driver: SharedWarmBackend = Arc::from(driver);
|
||||
let entry = Arc::new(TierDriverGeneration {
|
||||
tier_name: Arc::from(tier_name),
|
||||
tier_config: config.clone(),
|
||||
generation,
|
||||
config_fingerprint,
|
||||
backend_identity,
|
||||
exact_get_delete,
|
||||
driver: driver.clone(),
|
||||
reconciler: tokio::sync::OnceCell::new(),
|
||||
accepting: AtomicBool::new(true),
|
||||
active_leases: AtomicUsize::new(0),
|
||||
drained: tokio::sync::Notify::new(),
|
||||
@@ -9313,8 +9274,7 @@ mod tests {
|
||||
version_id: "v1".to_string(),
|
||||
tier_name: "COLD-A".to_string(),
|
||||
backend_identity: Some(current_identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
journal_store
|
||||
.insert_config_object(
|
||||
|
||||
@@ -73,21 +73,6 @@ pub enum TransitionCandidateProbe {
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct TransitionCandidateIdentity {
|
||||
pub transaction_id: uuid::Uuid,
|
||||
pub destination_id: [u8; 32],
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait TransitionCandidateReconciler {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait WarmBackend {
|
||||
async fn validate(&self) -> Result<(), std::io::Error> {
|
||||
@@ -204,20 +189,6 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
metadata.remove(key);
|
||||
}
|
||||
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
] {
|
||||
for key in [
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
|
||||
format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
|
||||
] {
|
||||
if let Some(value) = metadata.remove(&key) {
|
||||
metadata.insert(format!("x-amz-meta-{key}"), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
opts.user_metadata = metadata;
|
||||
opts
|
||||
}
|
||||
@@ -475,51 +446,6 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
pub(crate) async fn new_transition_candidate_reconciler(
|
||||
tier: &TierConfig,
|
||||
) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
|
||||
let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
|
||||
TierType::S3 => Box::new(
|
||||
WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::MinIO => Box::new(
|
||||
WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::RustFS => Box::new(
|
||||
WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::R2 => Box::new(
|
||||
WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
Ok(Some(reconciler))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -849,37 +775,6 @@ mod tests {
|
||||
assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"5a".repeat(32),
|
||||
);
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
] {
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}",
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
|
||||
)));
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}{suffix}",
|
||||
rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_requests_no_checksum_and_content_md5() {
|
||||
// Regression for rustfs/rustfs#4811: transition uploads must leave the
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
@@ -46,19 +45,6 @@ const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
|
||||
if remote_version.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let generation = remote_version
|
||||
.parse::<i64>()
|
||||
.map_err(|_| Error::new(ErrorKind::InvalidData, "GCS remote version is not a valid generation"))?;
|
||||
if generation <= 0 {
|
||||
return Err(Error::new(ErrorKind::InvalidData, "GCS remote version generation must be positive"));
|
||||
}
|
||||
Ok(Some(generation))
|
||||
}
|
||||
|
||||
pub struct WarmBackendGCS {
|
||||
pub client: Arc<Storage>,
|
||||
pub control: Arc<StorageControl>,
|
||||
@@ -119,10 +105,6 @@ impl WarmBackendGCS {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendGCS {
|
||||
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
|
||||
parse_generation(remote_version_id).map(|_| ())
|
||||
}
|
||||
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
@@ -153,9 +135,6 @@ impl WarmBackend for WarmBackendGCS {
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
}
|
||||
|
||||
// Honor the requested byte range so Range GETs on tiered objects return the exact
|
||||
// interval instead of the whole object (matches the s3/s3sdk/rustfs warm backends).
|
||||
@@ -185,15 +164,13 @@ impl WarmBackend for WarmBackendGCS {
|
||||
// gRPC v2 DeleteObject requires the bucket in resource-name form. Without this the
|
||||
// deleted tiered object was never removed from GCS (empty impl returned Ok), leaking
|
||||
// remote data forever.
|
||||
let mut req = self
|
||||
.control
|
||||
self.control
|
||||
.delete_object()
|
||||
.set_bucket(format!("projects/_/buckets/{}", self.bucket))
|
||||
.set_object(self.get_dest(object));
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
}
|
||||
req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
.set_object(self.get_dest(object))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -214,30 +191,6 @@ impl WarmBackend for WarmBackendGCS {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_generation;
|
||||
use std::io::ErrorKind;
|
||||
|
||||
#[test]
|
||||
fn generation_parser_preserves_exact_numeric_versions() {
|
||||
assert_eq!(parse_generation("").expect("empty generation means no version condition"), None);
|
||||
assert_eq!(parse_generation("1").expect("minimum generation should parse"), Some(1));
|
||||
assert_eq!(
|
||||
parse_generation(&i64::MAX.to_string()).expect("maximum generation should parse"),
|
||||
Some(i64::MAX)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generation_parser_rejects_unknown_or_non_positive_versions() {
|
||||
for value in ["unknown", "1.0", "-1", "0", "9223372036854775808"] {
|
||||
let err = parse_generation(value).expect_err("unknown generation must fail closed");
|
||||
assert_eq!(err.kind(), ErrorKind::InvalidData, "{value}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
|
||||
if err == nil {
|
||||
return nil
|
||||
|
||||
@@ -135,20 +135,6 @@ impl WarmBackend for WarmBackendMinIO {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -135,20 +135,6 @@ impl WarmBackend for WarmBackendR2 {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -132,20 +132,6 @@ impl WarmBackend for WarmBackendRustFS {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -29,7 +29,6 @@ use crate::client::{
|
||||
api_remove::{RemoveObjectOptions, RemoveObjectResult},
|
||||
api_s3_datatypes::ListVersionsResult,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
provider_versions::validate_remote_version_id,
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
@@ -37,10 +36,7 @@ use crate::error::ErrorResponse;
|
||||
use crate::error::error_resp_to_object_err;
|
||||
use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options,
|
||||
},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
@@ -193,188 +189,44 @@ impl WarmBackendS3 {
|
||||
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_versions(
|
||||
&self,
|
||||
object: &str,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
async fn list_transition_candidate_versions(&self, object: &str) -> Result<ListVersionsResult, std::io::Error> {
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &remote_object);
|
||||
opts.set("max-keys", "1000");
|
||||
|
||||
let mut key_marker = String::new();
|
||||
let mut version_id_marker = String::new();
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
loop {
|
||||
let versions = self
|
||||
.client
|
||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
|
||||
.await?;
|
||||
candidates.extend(&remote_object, &versions);
|
||||
if candidates.is_ambiguous() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
}
|
||||
if !versions.is_truncated {
|
||||
return classify_transition_candidates(candidates, bucket_versioning);
|
||||
}
|
||||
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_identity(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &remote_object);
|
||||
opts.set("max-keys", "1000");
|
||||
let mut key_marker = String::new();
|
||||
let mut version_id_marker = String::new();
|
||||
let mut matched_version = None;
|
||||
let mut saw_unproven_candidate = false;
|
||||
|
||||
loop {
|
||||
let versions = self
|
||||
.client
|
||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
|
||||
.await?;
|
||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||
let mut stat_opts = GetObjectOptions::default();
|
||||
stat_opts.version_id.clone_from(&version.version_id);
|
||||
let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
|
||||
let mut metadata = info.user_metadata;
|
||||
for (name, value) in &info.metadata {
|
||||
if (name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|
||||
|| name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
metadata.insert(name.as_str().to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
if transition_candidate_metadata_matches(&metadata, identity)? {
|
||||
if matched_version.is_some() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
}
|
||||
matched_version = Some(version.version_id.clone());
|
||||
} else {
|
||||
saw_unproven_candidate = true;
|
||||
}
|
||||
}
|
||||
if !versions.is_truncated {
|
||||
if matched_version.is_none() && saw_unproven_candidate {
|
||||
return Ok(TransitionCandidateProbe::Unsupported);
|
||||
}
|
||||
let candidates = TransitionCandidateVersions {
|
||||
version_id: matched_version,
|
||||
ambiguous: false,
|
||||
};
|
||||
return classify_transition_candidates(candidates, bucket_versioning);
|
||||
}
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
opts.set("prefix", &self.get_dest(object));
|
||||
opts.set("max-keys", "2");
|
||||
self.client.list_object_versions_query(&self.bucket, &opts, "", "", "").await
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_candidate_metadata_matches(
|
||||
metadata: &HashMap<String, String>,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<bool, std::io::Error> {
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
|
||||
};
|
||||
|
||||
let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
|
||||
let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
|
||||
if transaction_id.is_none() || destination_id.is_none() {
|
||||
if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
|
||||
|| contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"transition candidate identity metadata is empty or conflicting",
|
||||
));
|
||||
}
|
||||
return Ok(false);
|
||||
}
|
||||
let expected_transaction_id = identity.transaction_id.to_string();
|
||||
let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
|
||||
Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
|
||||
}
|
||||
|
||||
fn classify_transition_candidates(
|
||||
candidates: TransitionCandidateVersions,
|
||||
fn classify_transition_candidate_versions(
|
||||
remote_object: &str,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let probe = candidates.classify(bucket_versioning);
|
||||
if let TransitionCandidateProbe::VersionedPresent(version_id) = &probe {
|
||||
validate_remote_version_id(version_id)?;
|
||||
}
|
||||
Ok(probe)
|
||||
}
|
||||
|
||||
fn advance_version_markers(
|
||||
key_marker: &mut String,
|
||||
version_id_marker: &mut String,
|
||||
versions: &ListVersionsResult,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let next_markers = (&versions.next_key_marker, &versions.next_version_id_marker);
|
||||
if next_markers == (&*key_marker, &*version_id_marker) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"ListObjectVersions pagination markers did not advance",
|
||||
));
|
||||
}
|
||||
key_marker.clone_from(&versions.next_key_marker);
|
||||
version_id_marker.clone_from(&versions.next_version_id_marker);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TransitionCandidateVersions {
|
||||
version_id: Option<String>,
|
||||
ambiguous: bool,
|
||||
}
|
||||
|
||||
impl TransitionCandidateVersions {
|
||||
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
|
||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||
if self.version_id.is_some() {
|
||||
self.ambiguous = true;
|
||||
return;
|
||||
}
|
||||
self.version_id = Some(version.version_id.clone());
|
||||
}
|
||||
) -> TransitionCandidateProbe {
|
||||
if versions.is_truncated {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
fn is_ambiguous(&self) -> bool {
|
||||
self.ambiguous
|
||||
if versions.delete_markers.iter().any(|marker| marker.key == remote_object) {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
|
||||
if self.ambiguous {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
let Some(version_id) = self.version_id else {
|
||||
return TransitionCandidateProbe::Missing;
|
||||
};
|
||||
let mut exact_versions = versions.versions.iter().filter(|version| version.key == remote_object);
|
||||
let Some(version) = exact_versions.next() else {
|
||||
return TransitionCandidateProbe::Missing;
|
||||
};
|
||||
if exact_versions.next().is_some() {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
match bucket_versioning {
|
||||
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
|
||||
RemoteBucketVersioning::Suspended if version_id == "null" => TransitionCandidateProbe::VersionedPresent(version_id),
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version_id.is_empty() => {
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
|
||||
match bucket_versioning {
|
||||
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
|
||||
RemoteBucketVersioning::Suspended if version.version_id == "null" => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version.version_id.is_empty() => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -423,161 +275,72 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_pages(bucket_versioning: RemoteBucketVersioning, pages: &[ListVersionsResult]) -> TransitionCandidateProbe {
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
for page in pages {
|
||||
candidates.extend("archive/object", page);
|
||||
}
|
||||
candidates.classify(bucket_versioning)
|
||||
}
|
||||
|
||||
fn candidate_identity() -> TransitionCandidateIdentity {
|
||||
TransitionCandidateIdentity {
|
||||
transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
|
||||
destination_id: [0x5a; 32],
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
identity.transaction_id.to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(identity.destination_id),
|
||||
);
|
||||
metadata
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_identity_requires_exact_compatible_metadata() {
|
||||
let identity = candidate_identity();
|
||||
let metadata = candidate_metadata(identity);
|
||||
assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
|
||||
|
||||
let mut adjacent = metadata;
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut adjacent,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
);
|
||||
assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
|
||||
|
||||
assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
|
||||
let identity = candidate_identity();
|
||||
let mut metadata = candidate_metadata(identity);
|
||||
metadata.insert(
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
),
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
);
|
||||
assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_classifier_is_fail_closed() {
|
||||
assert_eq!(
|
||||
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[], &[], false)],),
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[("archive/object", "")], &[], false)],),
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::UnversionedPresent
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[list_versions(&[("archive/object", "version-a")], &[], false)],
|
||||
&list_versions(&[("archive/object", "version-a")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Suspended,
|
||||
&[list_versions(&[("archive/object", "null")], &[], false)],
|
||||
&list_versions(&[("archive/object", "null")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("null".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(RemoteBucketVersioning::Enabled, &[list_versions(&[("archive/object", "")], &[], false)],),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[list_versions(
|
||||
&[("archive/object", "version-a"), ("archive/object", "version-b")],
|
||||
&[],
|
||||
false,
|
||||
)],
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_reconciles_all_pages_and_ignores_delete_markers() {
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[
|
||||
list_versions(&[], &[("archive/object", "marker-a")], true),
|
||||
list_versions(&[("archive/object", "version-a"), ("archive/object-adjacent", "unrelated"),], &[], false,),
|
||||
],
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[
|
||||
list_versions(&[("archive/object", "version-a")], &[], true),
|
||||
list_versions(&[("archive/object", "version-b")], &[], false),
|
||||
],
|
||||
&list_versions(&[("archive/object", "version-a"), ("archive/object", "version-b")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[("archive/object", "marker-a")], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[], true),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_pagination_advances_both_markers() {
|
||||
let mut key_marker = "old-key".to_string();
|
||||
let mut version_id_marker = "old-version".to_string();
|
||||
let page = ListVersionsResult {
|
||||
next_key_marker: "next-key".to_string(),
|
||||
next_version_id_marker: "next-version".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
|
||||
.expect("new ListObjectVersions markers should advance pagination");
|
||||
assert_eq!(key_marker, "next-key");
|
||||
assert_eq!(version_id_marker, "next-version");
|
||||
|
||||
let err = advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
|
||||
.expect_err("repeated ListObjectVersions markers must fail closed");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_rejects_untrusted_version_ids() {
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
candidates.extend("archive/object", &list_versions(&[("archive/object", "version\ninjection")], &[], false));
|
||||
|
||||
let err = classify_transition_candidates(candidates, RemoteBucketVersioning::Enabled)
|
||||
.expect_err("control characters in listed version IDs must fail closed");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -634,7 +397,12 @@ impl WarmBackend for WarmBackendS3 {
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
self.probe_transition_candidate_versions(object, bucket_versioning).await
|
||||
let versions = self.list_transition_candidate_versions(object).await?;
|
||||
Ok(classify_transition_candidate_versions(
|
||||
&self.get_dest(object),
|
||||
bucket_versioning,
|
||||
&versions,
|
||||
))
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
@@ -646,16 +414,3 @@ impl WarmBackend for WarmBackendS3 {
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TransitionCandidateReconciler for WarmBackendS3 {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
self.probe_transition_candidate_identity(object, identity, bucket_versioning)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,10 +46,9 @@ use crate::diagnostics::get::{
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
||||
use crate::disk::{
|
||||
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
|
||||
PartTransactionAction, part_transaction_path,
|
||||
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
||||
part_transaction_path,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
@@ -548,8 +547,6 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo
|
||||
&& left.transitioned_objname == right.transitioned_objname
|
||||
&& left.transition_tier == right.transition_tier
|
||||
&& left.transition_version_id == right.transition_version_id
|
||||
&& left.transition_version == right.transition_version
|
||||
&& left.transition_version_state == right.transition_version_state
|
||||
&& left.expire_restored == right.expire_restored
|
||||
&& left.size == right.size
|
||||
&& left.mod_time == right.mod_time
|
||||
@@ -2988,48 +2985,29 @@ impl SetDisks {
|
||||
Self::rename_fanout_barrier(&object_for_fault, idx, rename_fanout_barrier_phase::CLEANUP).await;
|
||||
|
||||
if let Some(err) = Self::cleanup_injected_error(&object_for_fault, idx) {
|
||||
return (false, Some(err));
|
||||
return Some(err);
|
||||
}
|
||||
if let Some(disk) = disk {
|
||||
match disk
|
||||
.delete_data_dir(
|
||||
&bucket,
|
||||
&file_path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DataDirDeleteStatus::Deleted) => (false, None),
|
||||
Ok(DataDirDeleteStatus::Deferred) => (true, None),
|
||||
Err(err) => (false, Some(err)),
|
||||
}
|
||||
disk.delete(
|
||||
&bucket,
|
||||
&file_path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
} else {
|
||||
// `None` slot: ignored placeholder. It is not `attempted`, so
|
||||
// classification excludes it from residue regardless.
|
||||
(false, Some(DiskError::DiskNotFound))
|
||||
Some(DiskError::DiskNotFound)
|
||||
}
|
||||
})
|
||||
});
|
||||
let mut deferred = 0usize;
|
||||
let errs: Vec<Option<DiskError>> = join_all(futures)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|result| match result {
|
||||
Ok((was_deferred, err)) => {
|
||||
deferred += usize::from(was_deferred);
|
||||
err
|
||||
}
|
||||
Err(join_err) => Some(DiskError::other(format!("old data dir cleanup task failed: {join_err}"))),
|
||||
})
|
||||
.collect();
|
||||
let errs: Vec<Option<DiskError>> = join_all(futures).await.into_iter().map(map_cleanup_join_result).collect();
|
||||
|
||||
let mut cleanup = classify_old_data_dir_cleanup(&errs, &attempted, write_quorum);
|
||||
cleanup.deferred = deferred;
|
||||
cleanup.reclaimed = cleanup.reclaimed.saturating_sub(deferred);
|
||||
cleanup
|
||||
classify_old_data_dir_cleanup(&errs, &attempted, write_quorum)
|
||||
}
|
||||
|
||||
/// Test-only fault-injection seam for the old-data-dir cleanup path
|
||||
@@ -3120,20 +3098,6 @@ impl SetDisks {
|
||||
|
||||
rustfs_io_metrics::record_old_data_dir_cleanup(c.attempted, c.reclaimed, c.unreclaimed_disks.len(), c.below_quorum);
|
||||
|
||||
if c.deferred > 0 {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_WRITE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
old_data_dir = %old_dir,
|
||||
deferred = c.deferred,
|
||||
state = "old_data_cleanup_deferred",
|
||||
"Old data directory cleanup deferred for active snapshot leases"
|
||||
);
|
||||
}
|
||||
|
||||
if actions.warn {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -3953,9 +3917,9 @@ impl SetDisks {
|
||||
/// * The set of referenced data dirs is the UNION of `get_data_dirs()` across
|
||||
/// every online disk's `xl.meta`, so a dir named by *any* replica is kept.
|
||||
/// * If a disk holds the object directory but its `xl.meta` is missing or
|
||||
/// unparsable, the object is treated as degraded and unmarked data dirs are
|
||||
/// never removed. A data dir carrying a committed delete-transaction marker
|
||||
/// remains reclaimable after a downgrade/re-upgrade cleanup interruption.
|
||||
/// unparsable, the object is treated as degraded and NOTHING is removed —
|
||||
/// the unreadable copy could be the only one naming a live data dir, and a
|
||||
/// heal must run first.
|
||||
/// * Only subdirectories whose names parse as a UUID are ever considered;
|
||||
/// removal is non-recursive-safe via a recursive delete of the full stray
|
||||
/// data-dir path only.
|
||||
@@ -3970,7 +3934,7 @@ impl SetDisks {
|
||||
// physical UUID subdirectories present on each disk. Abort on any degraded
|
||||
// copy so a healable object is never stripped of a referenced data dir.
|
||||
let mut referenced: HashSet<Uuid> = HashSet::new();
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<(Uuid, bool)>)> = Vec::new();
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<Uuid>)> = Vec::new();
|
||||
let mut healthy_metas = 0usize;
|
||||
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
@@ -4006,22 +3970,6 @@ impl SetDisks {
|
||||
// to the orphan-dir / dangling-object heal paths.
|
||||
continue;
|
||||
}
|
||||
let mut committed = Vec::with_capacity(physical.len());
|
||||
for dir in physical {
|
||||
let data_dir = format!("{object}/{dir}");
|
||||
let committed_delete = disk.list_dir("", bucket, &data_dir, 0).await.is_ok_and(|entries| {
|
||||
entries.iter().any(|entry| {
|
||||
entry
|
||||
.strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
|
||||
.is_some_and(|transaction| Uuid::parse_str(transaction).is_ok())
|
||||
})
|
||||
});
|
||||
committed.push((dir, committed_delete));
|
||||
}
|
||||
if committed.iter().all(|(_, committed_delete)| *committed_delete) {
|
||||
per_disk_dirs.push((i, committed));
|
||||
continue;
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
bucket, object,
|
||||
@@ -4058,16 +4006,22 @@ impl SetDisks {
|
||||
|
||||
healthy_metas += 1;
|
||||
if !physical.is_empty() {
|
||||
per_disk_dirs.push((i, physical.into_iter().map(|dir| (dir, false)).collect()));
|
||||
per_disk_dirs.push((i, physical));
|
||||
}
|
||||
}
|
||||
|
||||
// No healthy metadata anywhere: this is not a live object, so surplus dirs
|
||||
// (if any) belong to the dangling-object heal path, not here.
|
||||
if healthy_metas == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Phase 2: delete every physical data dir not referenced by the union.
|
||||
let mut removed = 0usize;
|
||||
for (i, physical) in per_disk_dirs {
|
||||
let Some(disk) = disks[i].as_ref() else { continue };
|
||||
for (dir, committed_delete) in physical {
|
||||
if referenced.contains(&dir) || (healthy_metas == 0 && !committed_delete) {
|
||||
for dir in physical {
|
||||
if referenced.contains(&dir) {
|
||||
continue;
|
||||
}
|
||||
let stray = format!("{object}/{dir}");
|
||||
@@ -4175,9 +4129,6 @@ pub(in crate::set_disk) struct OldDataDirCleanup {
|
||||
/// Number of attempted disks that returned `Ok` or a not-found variant
|
||||
/// (a missing dir == already reclaimed).
|
||||
pub reclaimed: usize,
|
||||
/// Number of attempted disks that retained the directory for an active
|
||||
/// snapshot lease and registered it for deletion after the final release.
|
||||
pub deferred: usize,
|
||||
/// Indices of attempted disks that failed with a non-ignored, non-not-found
|
||||
/// error (including task panic/cancel). This is the residue that actually
|
||||
/// leaks and drives the leak metric + heal enqueue.
|
||||
@@ -4240,7 +4191,6 @@ fn classify_old_data_dir_cleanup(errs: &[Option<DiskError>], attempted: &[bool],
|
||||
OldDataDirCleanup {
|
||||
attempted: attempted_count,
|
||||
reclaimed,
|
||||
deferred: 0,
|
||||
unreclaimed_disks,
|
||||
below_quorum,
|
||||
}
|
||||
@@ -5181,40 +5131,6 @@ mod tests {
|
||||
drop((disk1, disk2));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn commit_cleanup_reports_and_releases_deferred_snapshot_data_dirs() {
|
||||
let bucket = "cleanup-lease-bucket";
|
||||
let object = "cleanup-lease-object";
|
||||
let old_data_dir = "11111111-1111-1111-1111-111111111111";
|
||||
let committed_data_dir = "22222222-2222-2222-2222-222222222222";
|
||||
let data_dir_path = format!("{object}/{old_data_dir}");
|
||||
let shard_path = format!("{data_dir_path}/part.1");
|
||||
let (_dir1, disk1) = read_multiple_test_disk(bucket, &[(&shard_path, b"one".as_slice())]).await;
|
||||
let set = io_primitives_test_set(vec![Some(disk1.clone())], 0).await;
|
||||
let lease = disk1
|
||||
.acquire_snapshot_lease(bucket, &data_dir_path)
|
||||
.await
|
||||
.expect("snapshot lease should be acquired before cleanup");
|
||||
|
||||
let cleanup = set
|
||||
.commit_rename_data_dir(&[Some(disk1.clone())], bucket, object, old_data_dir, committed_data_dir, 1)
|
||||
.await;
|
||||
assert_eq!(cleanup.attempted, 1);
|
||||
assert_eq!(cleanup.reclaimed, 0);
|
||||
assert_eq!(cleanup.deferred, 1);
|
||||
assert!(cleanup.unreclaimed_disks.is_empty());
|
||||
disk1
|
||||
.read_all(bucket, &shard_path)
|
||||
.await
|
||||
.expect("deferred cleanup must leave later shard opens available");
|
||||
|
||||
disk1
|
||||
.release_snapshot_lease(bucket, &data_dir_path, lease)
|
||||
.await
|
||||
.expect("final lease release should reclaim the old data directory");
|
||||
assert!(matches!(disk1.read_all(bucket, &shard_path).await, Err(DiskError::FileNotFound)));
|
||||
}
|
||||
|
||||
/// Isolation guard: an armed barrier / observed object only affects its own
|
||||
/// object. A fan-out for a different (unobserved, unarmed) object must not be
|
||||
/// paused and must not accrue any tracked task count — so concurrent tests
|
||||
|
||||
@@ -555,7 +555,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
|
||||
fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
Self::update_file_info_quorum_hash(&mut hasher, meta);
|
||||
let digest = hasher.finalize();
|
||||
@@ -578,13 +578,6 @@ impl SetDisks {
|
||||
Self::update_hash_str(hasher, &meta.transition_tier);
|
||||
Self::update_hash_str(hasher, &meta.transitioned_objname);
|
||||
Self::update_hash_optional_uuid(hasher, meta.transition_version_id);
|
||||
Self::update_hash_optional_str(hasher, meta.transition_version.as_deref());
|
||||
hasher.update([match meta.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => 0,
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled => 1,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull => 2,
|
||||
rustfs_filemeta::TransitionVersionState::Exact => 3,
|
||||
}]);
|
||||
Self::update_hash_optional_u32(hasher, meta.mode);
|
||||
Self::update_hash_optional_u64(hasher, meta.written_by_version);
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -92,69 +92,6 @@ struct PartFailureSummary {
|
||||
bitrot_failure: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RecoverableMetaCandidate {
|
||||
identity: [u8; 32],
|
||||
file_info: FileInfo,
|
||||
data_count: usize,
|
||||
local_payload: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum DanglingDeleteSafety {
|
||||
UnsafeToDelete,
|
||||
NoRecoverableCandidate,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DanglingCheckPartsFailure {
|
||||
key: DanglingCheckPartsFailureKey,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type DanglingCheckPartsFailureKey = (String, String, usize);
|
||||
|
||||
#[cfg(test)]
|
||||
type DanglingCheckPartsFailures = HashMap<DanglingCheckPartsFailureKey, DiskError>;
|
||||
|
||||
#[cfg(test)]
|
||||
fn dangling_check_parts_failures() -> &'static std::sync::Mutex<DanglingCheckPartsFailures> {
|
||||
static FAILURES: std::sync::OnceLock<std::sync::Mutex<DanglingCheckPartsFailures>> = std::sync::OnceLock::new();
|
||||
FAILURES.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl DanglingCheckPartsFailure {
|
||||
fn install(bucket: &str, object: &str, disk_index: usize, error: DiskError) -> Self {
|
||||
let key = (bucket.to_string(), object.to_string(), disk_index);
|
||||
let previous = dangling_check_parts_failures()
|
||||
.lock()
|
||||
.expect("dangling check-parts failure registry should not poison")
|
||||
.insert(key.clone(), error);
|
||||
assert!(previous.is_none(), "dangling check-parts failure already installed");
|
||||
Self { key }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for DanglingCheckPartsFailure {
|
||||
fn drop(&mut self) {
|
||||
dangling_check_parts_failures()
|
||||
.lock()
|
||||
.expect("dangling check-parts failure registry should not poison")
|
||||
.remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn injected_dangling_check_parts_error(bucket: &str, object: &str, disk_index: usize) -> Option<DiskError> {
|
||||
dangling_check_parts_failures()
|
||||
.lock()
|
||||
.expect("dangling check-parts failure registry should not poison")
|
||||
.get(&(bucket.to_string(), object.to_string(), disk_index))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn first_unhealthy_part_summary(
|
||||
data_errs_by_part: &HashMap<usize, Vec<usize>>,
|
||||
parts: &[ObjectPartInfo],
|
||||
@@ -188,17 +125,39 @@ impl SetDisks {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, true)).await
|
||||
// `allow_meta_regen` is true on the first pass: a version whose data shards
|
||||
// physically survive (>= data_blocks) but whose xl.meta fell below
|
||||
// read-quorum is RESCUED (missing xl.meta regenerated) rather than
|
||||
// dangling-deleted. The re-drive after a rescue sets it false so the
|
||||
// regeneration can happen at most once (no unbounded recursion).
|
||||
Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, true)).await
|
||||
}
|
||||
|
||||
/// Best-effort orphan-data-dir reclaim for an object that is healthy on this
|
||||
/// set. Wraps [`Self::reclaim_orphan_data_dirs`] with the shared logging so
|
||||
/// both `heal_object` exits — the already-healthy early return and the
|
||||
/// post-heal tail — reclaim identically. Never fails the heal: delete errors
|
||||
/// are logged and swallowed. Callers must gate this on `!opts.dry_run`.
|
||||
async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
|
||||
match self.reclaim_orphan_data_dirs(bucket, object).await {
|
||||
Ok(removed) if removed > 0 => {
|
||||
info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
async fn heal_object_with_explicit_version_regen(
|
||||
async fn heal_object_with_regen(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_explicit_version_regen: bool,
|
||||
allow_meta_regen: bool,
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
info!(?opts, "Starting heal_object");
|
||||
|
||||
@@ -395,6 +354,22 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
// DATA-SAFETY GUARD (backlog#920, decision 1): before any
|
||||
// dangling delete, if the version's DATA shards physically
|
||||
// survive on >= data_blocks disks it is RECONSTRUCTABLE.
|
||||
// Regenerate the missing xl.meta from a surviving valid
|
||||
// FileInfo and re-drive the heal instead of destroying a
|
||||
// recoverable version. Torn writes (< data_blocks data
|
||||
// shards) fall through to the existing dangling behavior.
|
||||
if cannot_heal
|
||||
&& allow_meta_regen
|
||||
&& self
|
||||
.try_regenerate_recoverable_meta(bucket, object, &parts_metadata, &errs, &disks)
|
||||
.await?
|
||||
{
|
||||
return Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, false)).await;
|
||||
}
|
||||
|
||||
if cannot_heal {
|
||||
let total_disks = parts_metadata.len();
|
||||
let healthy_count = total_disks.saturating_sub(disks_to_heal_count);
|
||||
@@ -447,20 +422,6 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
// `disks_with_all_parts` normalizes conflicting entries
|
||||
// in `parts_metadata` to defaults. Re-read only before
|
||||
// destructive cleanup so the guard sees every original
|
||||
// identity.
|
||||
let (delete_guard_metadata, delete_guard_errs) =
|
||||
Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?;
|
||||
if self
|
||||
.dangling_delete_safety(bucket, object, &delete_guard_metadata, &delete_guard_errs, &disks)
|
||||
.await?
|
||||
== DanglingDeleteSafety::UnsafeToDelete
|
||||
{
|
||||
return Ok((result, Some(cannot_heal_err)));
|
||||
}
|
||||
|
||||
// Allow for dangling deletes, on versions that have DataDir missing etc.
|
||||
// this would end up restoring the correct readable versions.
|
||||
return match self
|
||||
@@ -876,25 +837,17 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if allow_explicit_version_regen
|
||||
&& !version_id.is_empty()
|
||||
// DATA-SAFETY GUARD (backlog#920, decision 1): meta quorum failed,
|
||||
// but the version's DATA may still physically survive on enough
|
||||
// disks (xl.meta lost on > parity disks while part files remain).
|
||||
// Rescue it by regenerating the missing xl.meta and re-driving heal
|
||||
// instead of dangling-deleting a reconstructable version.
|
||||
if allow_meta_regen
|
||||
&& self
|
||||
.try_regenerate_explicit_version_meta(bucket, object, version_id, &parts_metadata, &errs, &disks)
|
||||
.try_regenerate_recoverable_meta(bucket, object, &parts_metadata, &errs, &disks)
|
||||
.await?
|
||||
{
|
||||
return Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, false)).await;
|
||||
}
|
||||
|
||||
if self
|
||||
.dangling_delete_safety(bucket, object, &parts_metadata, &errs, &disks)
|
||||
.await?
|
||||
== DanglingDeleteSafety::UnsafeToDelete
|
||||
{
|
||||
return Ok((
|
||||
self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
|
||||
.await,
|
||||
Some(err),
|
||||
));
|
||||
return Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, false)).await;
|
||||
}
|
||||
|
||||
let data_errs_by_part = HashMap::new();
|
||||
@@ -930,226 +883,129 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
async fn try_regenerate_explicit_version_meta(
|
||||
/// backlog#920 (decision 1): rescue a version that meta-quorum logic would
|
||||
/// otherwise dangling-DELETE, when its DATA is still reconstructable.
|
||||
///
|
||||
/// Returns `Ok(true)` if the version was rescued (missing xl.meta regenerated
|
||||
/// on at least one disk, so a re-driven heal can reconstruct it), `Ok(false)`
|
||||
/// to fall through to the existing dangling-delete behavior.
|
||||
///
|
||||
/// Recoverability is computed by physically probing part files across ALL
|
||||
/// disks in the set with `check_parts` — including disks whose xl.meta is
|
||||
/// absent (a lost xl.meta does not lose the sibling `part.*` data). If at
|
||||
/// least `data_blocks` disks hold every part of a surviving valid FileInfo,
|
||||
/// the object is EC-reconstructable, so we regenerate that FileInfo's xl.meta
|
||||
/// on every disk whose metadata is absent (via `write_metadata`, which merges
|
||||
/// into any existing xl.meta). Delete markers, remote/transitioned versions,
|
||||
/// and genuine torn writes (< `data_blocks` surviving data shards) are NOT
|
||||
/// rescued — they keep the current dangling-delete-after-grace behavior, so no
|
||||
/// regression on those paths.
|
||||
async fn try_regenerate_recoverable_meta(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
parts_metadata: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> disk::error::Result<bool> {
|
||||
let Ok(version_id) = Uuid::parse_str(version_id) else {
|
||||
// A surviving valid, non-deleted, non-remote data FileInfo to rebuild from.
|
||||
let Some(surviving) = parts_metadata
|
||||
.iter()
|
||||
.find(|fi| fi.has_valid_erasure_geometry() && !fi.deleted && !fi.is_remote())
|
||||
.cloned()
|
||||
else {
|
||||
return Ok(false);
|
||||
};
|
||||
let candidates = parts_metadata
|
||||
.iter()
|
||||
.zip(errs.iter())
|
||||
.filter_map(|(file_info, err)| {
|
||||
(err.is_none()
|
||||
&& file_info_is_valid_for_metadata(file_info)
|
||||
&& file_info.version_id == Some(version_id)
|
||||
&& file_info.has_valid_erasure_geometry()
|
||||
&& !file_info.deleted
|
||||
&& !file_info.is_remote()
|
||||
&& file_info.data_dir.is_some()
|
||||
&& !file_info.parts.is_empty()
|
||||
&& file_info.erasure.data_blocks > 0
|
||||
&& file_info
|
||||
.erasure
|
||||
.data_blocks
|
||||
.checked_add(file_info.erasure.parity_blocks)
|
||||
.is_some_and(|shards| shards == disks.len()))
|
||||
.then_some(file_info)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let Some(candidate) = candidates.first().copied() else {
|
||||
|
||||
// Without a data_dir + parts there is no data to prove recoverable.
|
||||
if surviving.data_dir.is_none() || surviving.parts.is_empty() {
|
||||
return Ok(false);
|
||||
};
|
||||
let identity = Self::file_info_quorum_hash(candidate);
|
||||
if candidates
|
||||
.iter()
|
||||
.any(|file_info| Self::file_info_quorum_hash(file_info) != identity)
|
||||
{
|
||||
}
|
||||
let data_blocks = surviving.erasure.data_blocks;
|
||||
if data_blocks == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Physically probe part presence on EVERY online disk using the surviving
|
||||
// FileInfo's data_dir/parts. `check_parts` stats `object/<data_dir>/part.N`
|
||||
// directly, so it counts disks that still hold the data even if their
|
||||
// xl.meta was deleted.
|
||||
let mut available = 0usize;
|
||||
for disk in disks {
|
||||
let Some(disk) = disk else {
|
||||
return Ok(false);
|
||||
};
|
||||
match disk.check_parts(bucket, object, candidate).await {
|
||||
Ok(response)
|
||||
if !response.results.is_empty() && response.results.iter().all(|result| *result == CHECK_PART_SUCCESS) =>
|
||||
{
|
||||
available += 1;
|
||||
}
|
||||
Ok(_)
|
||||
| Err(
|
||||
DiskError::FileNotFound
|
||||
| DiskError::FileVersionNotFound
|
||||
| DiskError::PathNotFound
|
||||
| DiskError::VolumeNotFound,
|
||||
) => {}
|
||||
Err(_) => return Ok(false),
|
||||
for disk in disks.iter().flatten() {
|
||||
if let Ok(resp) = disk.check_parts(bucket, object, &surviving).await
|
||||
&& !resp.results.is_empty()
|
||||
&& resp.results.iter().all(|r| *r == CHECK_PART_SUCCESS)
|
||||
{
|
||||
available += 1;
|
||||
}
|
||||
}
|
||||
if available < candidate.erasure.data_blocks {
|
||||
|
||||
// Torn write: fewer than data_blocks surviving data shards is genuinely
|
||||
// unrecoverable — preserve the current dangling behavior (no resurrection).
|
||||
if available < data_blocks {
|
||||
debug!(
|
||||
bucket,
|
||||
object,
|
||||
available,
|
||||
data_blocks,
|
||||
"heal_object: version not reconstructable (torn write), keeping dangling behavior"
|
||||
);
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Reconstructable: regenerate the surviving xl.meta on every disk whose
|
||||
// metadata is absent so the version regains read-quorum. Each disk gets its
|
||||
// OWN shard index: the disk at physical position `index` holds shard
|
||||
// `distribution[index]` (mirrors `shuffle_disks` + the write path's
|
||||
// `erasure.index = shuffled_pos + 1`). Copying the surviving disk's index
|
||||
// verbatim would write an inconsistent xl.meta that the re-heal then treats
|
||||
// as corrupt.
|
||||
let distribution = &surviving.erasure.distribution;
|
||||
let mut wrote = 0usize;
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let Some(disk) = disk else {
|
||||
return Ok(false);
|
||||
};
|
||||
let metadata_absent = matches!(
|
||||
let Some(disk) = disk else { continue };
|
||||
let meta_absent = matches!(
|
||||
errs.get(index).and_then(Option::as_ref),
|
||||
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
||||
);
|
||||
if !metadata_absent {
|
||||
) || !parts_metadata.get(index).map(FileInfo::is_valid).unwrap_or(false);
|
||||
if !meta_absent {
|
||||
continue;
|
||||
}
|
||||
let Some(&shard_index) = candidate.erasure.distribution.get(index) else {
|
||||
return Ok(false);
|
||||
// Without a known shard index for this position we cannot write a
|
||||
// consistent xl.meta; leave it for the normal heal to reconstruct.
|
||||
let Some(&shard_index) = distribution.get(index) else {
|
||||
continue;
|
||||
};
|
||||
let mut regenerated = candidate.clone();
|
||||
regenerated.fresh = false;
|
||||
regenerated.erasure.index = shard_index;
|
||||
match disk.write_metadata("", bucket, object, regenerated).await {
|
||||
let mut regen = surviving.clone();
|
||||
regen.fresh = false; // merge into any existing xl.meta on the disk
|
||||
regen.erasure.index = shard_index;
|
||||
match disk.write_metadata("", bucket, object, regen).await {
|
||||
Ok(()) => wrote += 1,
|
||||
Err(error) => {
|
||||
Err(e) => {
|
||||
warn!(
|
||||
bucket,
|
||||
object,
|
||||
disk_index = index,
|
||||
error = %error,
|
||||
"failed to regenerate recoverable xl.meta"
|
||||
error = %e,
|
||||
"heal_object: failed to regenerate recoverable xl.meta on disk"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(wrote > 0)
|
||||
}
|
||||
|
||||
/// Best-effort orphan-data-dir reclaim for an object that is healthy on this
|
||||
/// set. Wraps [`Self::reclaim_orphan_data_dirs`] with the shared logging so
|
||||
/// both `heal_object` exits — the already-healthy early return and the
|
||||
/// post-heal tail — reclaim identically. Never fails the heal: delete errors
|
||||
/// are logged and swallowed. Callers must gate this on `!opts.dry_run`.
|
||||
async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
|
||||
match self.reclaim_orphan_data_dirs(bucket, object).await {
|
||||
Ok(removed) if removed > 0 => {
|
||||
info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Prevent dangling cleanup when surviving state cannot prove that deletion
|
||||
/// is safe. Part presence proves only recoverability, never commit: the write
|
||||
/// path can durably rename data before xl.meta is committed.
|
||||
async fn dangling_delete_safety(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
parts_metadata: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> disk::error::Result<DanglingDeleteSafety> {
|
||||
if disks.iter().any(Option::is_none)
|
||||
|| errs.iter().flatten().any(|err| {
|
||||
!matches!(
|
||||
err,
|
||||
DiskError::FileNotFound
|
||||
| DiskError::FileVersionNotFound
|
||||
| DiskError::PathNotFound
|
||||
| DiskError::VolumeNotFound
|
||||
)
|
||||
})
|
||||
{
|
||||
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
||||
if wrote == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut candidates = Vec::<RecoverableMetaCandidate>::with_capacity(parts_metadata.len());
|
||||
for (fi, err) in parts_metadata.iter().zip(errs.iter()) {
|
||||
if err.is_some() || !file_info_is_valid_for_metadata(fi) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let identity = Self::file_info_quorum_hash(fi);
|
||||
if !candidates.iter().any(|candidate| candidate.identity == identity) {
|
||||
let local_payload = fi.has_valid_erasure_geometry()
|
||||
&& !fi.deleted
|
||||
&& !fi.is_remote()
|
||||
&& fi.data_dir.is_some()
|
||||
&& !fi.parts.is_empty()
|
||||
&& fi.erasure.data_blocks > 0
|
||||
&& fi
|
||||
.erasure
|
||||
.data_blocks
|
||||
.checked_add(fi.erasure.parity_blocks)
|
||||
.is_some_and(|shards| shards == disks.len());
|
||||
candidates.push(RecoverableMetaCandidate {
|
||||
identity,
|
||||
file_info: fi.clone(),
|
||||
data_count: 0,
|
||||
local_payload,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if candidates
|
||||
.iter()
|
||||
.any(|candidate| candidate.file_info.deleted || candidate.file_info.is_remote())
|
||||
|| candidates.len() > 1
|
||||
{
|
||||
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
||||
}
|
||||
|
||||
for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
|
||||
for (disk_index, disk) in disks.iter().enumerate() {
|
||||
let Some(disk) = disk else {
|
||||
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
||||
};
|
||||
#[cfg(test)]
|
||||
let check_result = match injected_dangling_check_parts_error(bucket, object, disk_index) {
|
||||
Some(error) => Err(error),
|
||||
None => disk.check_parts(bucket, object, &candidate.file_info).await,
|
||||
};
|
||||
#[cfg(not(test))]
|
||||
let check_result = disk.check_parts(bucket, object, &candidate.file_info).await;
|
||||
|
||||
match check_result {
|
||||
Ok(resp) if !resp.results.is_empty() && resp.results.iter().all(|result| *result == CHECK_PART_SUCCESS) => {
|
||||
candidate.data_count += 1;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(
|
||||
DiskError::FileNotFound
|
||||
| DiskError::FileVersionNotFound
|
||||
| DiskError::PathNotFound
|
||||
| DiskError::VolumeNotFound,
|
||||
) => {}
|
||||
Err(_) => return Ok(DanglingDeleteSafety::UnsafeToDelete),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(
|
||||
if candidates
|
||||
.iter()
|
||||
.any(|candidate| candidate.local_payload && candidate.data_count >= candidate.file_info.erasure.data_blocks)
|
||||
{
|
||||
DanglingDeleteSafety::UnsafeToDelete
|
||||
} else {
|
||||
DanglingDeleteSafety::NoRecoverableCandidate
|
||||
},
|
||||
)
|
||||
info!(
|
||||
bucket,
|
||||
object,
|
||||
available,
|
||||
data_blocks,
|
||||
regenerated_meta_disks = wrote,
|
||||
"heal_object: rescued reconstructable sub-quorum version by regenerating xl.meta"
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn heal_object_dir_locked(
|
||||
@@ -1537,7 +1393,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
|
||||
#[cfg(test)]
|
||||
mod heal_result_report_tests {
|
||||
use super::{DanglingCheckPartsFailure, DanglingDeleteSafety, SetDisks};
|
||||
use super::SetDisks;
|
||||
use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::error::DiskError;
|
||||
@@ -1549,12 +1405,10 @@ mod heal_result_report_tests {
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::{config::storageclass, store::init_format::save_format_file};
|
||||
use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
|
||||
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
|
||||
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo};
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::RwLock;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn real_disk() -> (TempDir, Endpoint, DiskStore) {
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
@@ -1592,68 +1446,6 @@ mod heal_result_report_tests {
|
||||
.await
|
||||
}
|
||||
|
||||
fn meta_regen_test_fileinfo(object: &str, data_dir: Uuid, mod_time: i64, disk_index: usize) -> FileInfo {
|
||||
let mut fi = FileInfo::new(object, 2, 2);
|
||||
fi.data_dir = Some(data_dir);
|
||||
fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(mod_time).expect("test timestamp should parse"));
|
||||
fi.size = 1;
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.erasure.index = fi.erasure.distribution[disk_index];
|
||||
fi
|
||||
}
|
||||
|
||||
async fn meta_regen_test_set(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data_dirs: &[(Uuid, usize)],
|
||||
) -> (Vec<TempDir>, Arc<SetDisks>, Vec<Option<DiskStore>>) {
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let (temp_dir, endpoint, disk) = real_disk().await;
|
||||
disk.make_volume(bucket).await.expect("test bucket should be created");
|
||||
for (data_dir, shard_count) in data_dirs {
|
||||
if disk_index >= *shard_count {
|
||||
continue;
|
||||
}
|
||||
let part_dir = temp_dir.path().join(bucket).join(object).join(data_dir.to_string());
|
||||
tokio::fs::create_dir_all(&part_dir)
|
||||
.await
|
||||
.expect("test data directory should be created");
|
||||
tokio::fs::write(part_dir.join("part.1"), [1u8; 2])
|
||||
.await
|
||||
.expect("test data shard should be written");
|
||||
}
|
||||
temp_dirs.push(temp_dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
let set = set_disks_with(disks.clone(), endpoints, 2).await;
|
||||
(temp_dirs, set, disks)
|
||||
}
|
||||
|
||||
async fn seed_meta_regen_test_metadata(
|
||||
disks: &[Option<DiskStore>],
|
||||
disk_index: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
file_info: &FileInfo,
|
||||
) {
|
||||
disks[disk_index]
|
||||
.as_ref()
|
||||
.expect("metadata test disk should be online")
|
||||
.write_metadata("", bucket, object, file_info.clone())
|
||||
.await
|
||||
.expect("test metadata should be written");
|
||||
}
|
||||
|
||||
async fn formatted_single_disk_no_parity_set() -> (TempDir, Arc<SetDisks>) {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
@@ -1961,312 +1753,6 @@ mod heal_result_report_tests {
|
||||
assert_eq!(result.before.drives[3].state, DriveState::Ok.to_string());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_delete_guard_preserves_conflicting_identities_without_writing_metadata() {
|
||||
let bucket = "bucket-delete-guard-conflict";
|
||||
let object = "object.bin";
|
||||
let old_data_dir = Uuid::parse_str("99999999-9999-9999-9999-999999999999").expect("old data dir should parse");
|
||||
let new_data_dir = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa").expect("new data dir should parse");
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(old_data_dir, 4), (new_data_dir, 2)]).await;
|
||||
let version_id = Uuid::parse_str("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb").expect("version id should parse");
|
||||
let mut metadata = vec![
|
||||
meta_regen_test_fileinfo(object, old_data_dir, 9, 0),
|
||||
meta_regen_test_fileinfo(object, new_data_dir, 10, 1),
|
||||
FileInfo::default(),
|
||||
FileInfo::default(),
|
||||
];
|
||||
metadata[0].version_id = Some(version_id);
|
||||
metadata[1].version_id = Some(version_id);
|
||||
assert_eq!(
|
||||
metadata[0].version_id, metadata[1].version_id,
|
||||
"the conflicting candidates must share one version id"
|
||||
);
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata[0]).await;
|
||||
seed_meta_regen_test_metadata(&disks, 1, bucket, object, &metadata[1]).await;
|
||||
let errs = vec![None, None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)];
|
||||
|
||||
assert!(
|
||||
set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
|
||||
.await
|
||||
.expect("conflicting identities should be classified")
|
||||
== DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
let reversed = vec![
|
||||
metadata[1].clone(),
|
||||
metadata[0].clone(),
|
||||
FileInfo::default(),
|
||||
FileInfo::default(),
|
||||
];
|
||||
assert!(
|
||||
set.dangling_delete_safety(bucket, object, &reversed, &errs, &disks)
|
||||
.await
|
||||
.expect("reversed identities should be classified")
|
||||
== DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
let version_id = version_id.to_string();
|
||||
assert!(
|
||||
!set.try_regenerate_explicit_version_meta(bucket, object, &version_id, &metadata, &errs, &disks)
|
||||
.await
|
||||
.expect("conflicting explicit-version candidates should be rejected"),
|
||||
"an explicit version must not select between conflicting metadata identities"
|
||||
);
|
||||
for disk_index in [2, 3] {
|
||||
assert!(
|
||||
matches!(
|
||||
disks[disk_index]
|
||||
.as_ref()
|
||||
.expect("test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"the delete guard must not manufacture metadata on missing disks"
|
||||
);
|
||||
}
|
||||
let old = disks[0]
|
||||
.as_ref()
|
||||
.expect("first test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("old metadata should remain readable");
|
||||
let new = disks[1]
|
||||
.as_ref()
|
||||
.expect("second test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("new metadata should remain readable");
|
||||
assert_eq!(old.data_dir, Some(old_data_dir));
|
||||
assert_eq!(new.data_dir, Some(new_data_dir));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_meta_quorum_failure_preserves_reconstructable_uncommitted_candidate() {
|
||||
let bucket = "bucket-delete-guard-reconstructable";
|
||||
let object = "object.bin";
|
||||
let data_dir = Uuid::parse_str("33333333-3333-3333-3333-333333333333").expect("data dir should parse");
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
|
||||
let metadata = [
|
||||
meta_regen_test_fileinfo(object, data_dir, 3, 0),
|
||||
FileInfo::default(),
|
||||
FileInfo::default(),
|
||||
FileInfo::default(),
|
||||
];
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata[0]).await;
|
||||
let (observed_metadata, observed_errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", true, true, false)
|
||||
.await
|
||||
.expect("test metadata should be readable across the set");
|
||||
assert_eq!(
|
||||
set.dangling_delete_safety(bucket, object, &observed_metadata, &observed_errs, &disks)
|
||||
.await
|
||||
.expect("observed reconstructable candidate should be classified"),
|
||||
DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
|
||||
let (_, err) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("unsafe dangling state should be reported without deletion");
|
||||
assert_eq!(err, Some(DiskError::FileNotFound));
|
||||
let surviving = disks[0]
|
||||
.as_ref()
|
||||
.expect("first test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("the only metadata copy must be preserved");
|
||||
assert_eq!(surviving.data_dir, Some(data_dir));
|
||||
assert!(
|
||||
matches!(
|
||||
disks[1]
|
||||
.as_ref()
|
||||
.expect("second test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"the delete guard must not propagate metadata"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_meta_quorum_failure_preserves_candidate_when_required_shard_disk_is_offline() {
|
||||
let bucket = "bucket-delete-guard-offline";
|
||||
let object = "object.bin";
|
||||
let data_dir = Uuid::parse_str("44444444-4444-4444-4444-444444444444").expect("data dir should parse");
|
||||
let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
|
||||
let metadata = meta_regen_test_fileinfo(object, data_dir, 4, 0);
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
|
||||
set.disks.write().await[1] = None;
|
||||
|
||||
let (_, err) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("offline shard state should be reported without deletion");
|
||||
assert_eq!(err, Some(DiskError::FileNotFound));
|
||||
let surviving = disks[0]
|
||||
.as_ref()
|
||||
.expect("first test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("offline uncertainty must preserve the surviving metadata");
|
||||
assert_eq!(surviving.data_dir, Some(data_dir));
|
||||
assert!(
|
||||
temp_dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1")
|
||||
.is_file(),
|
||||
"offline uncertainty must preserve the last online shard"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_meta_quorum_failure_preserves_candidate_when_part_probe_times_out() {
|
||||
let bucket = "bucket-delete-guard-timeout";
|
||||
let object = "object.bin";
|
||||
let data_dir = Uuid::parse_str("55555555-5555-5555-5555-555555555555").expect("data dir should parse");
|
||||
let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
|
||||
let metadata = meta_regen_test_fileinfo(object, data_dir, 5, 0);
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
|
||||
let _failure = DanglingCheckPartsFailure::install(bucket, object, 1, DiskError::Timeout);
|
||||
|
||||
let (_, err) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("part probe timeout should be reported without deletion");
|
||||
assert_eq!(err, Some(DiskError::FileNotFound));
|
||||
let surviving = disks[0]
|
||||
.as_ref()
|
||||
.expect("first test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("probe uncertainty must preserve the surviving metadata");
|
||||
assert_eq!(surviving.data_dir, Some(data_dir));
|
||||
assert!(
|
||||
temp_dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1")
|
||||
.is_file(),
|
||||
"probe uncertainty must preserve the last confirmed shard"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_delete_guard_ignores_set_incompatible_geometry() {
|
||||
let bucket = "bucket-delete-guard-short-geometry";
|
||||
let object = "object.bin";
|
||||
let data_dir = Uuid::parse_str("abababab-abab-abab-abab-abababababab").expect("data dir should parse");
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 1)]).await;
|
||||
let mut candidate = FileInfo::new(object, 1, 0);
|
||||
candidate.data_dir = Some(data_dir);
|
||||
candidate.mod_time = Some(OffsetDateTime::from_unix_timestamp(18).expect("timestamp should parse"));
|
||||
candidate.size = 1;
|
||||
candidate.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
candidate.erasure.index = candidate.erasure.distribution[0];
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &candidate).await;
|
||||
let metadata = vec![candidate, FileInfo::default(), FileInfo::default(), FileInfo::default()];
|
||||
let errs = vec![
|
||||
None,
|
||||
Some(DiskError::FileNotFound),
|
||||
Some(DiskError::FileNotFound),
|
||||
Some(DiskError::FileNotFound),
|
||||
];
|
||||
|
||||
assert!(
|
||||
set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
|
||||
.await
|
||||
.expect("set-incompatible geometry should be classified")
|
||||
== DanglingDeleteSafety::NoRecoverableCandidate
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_delete_guard_preserves_delete_marker_and_remote_metadata() {
|
||||
let bucket = "bucket-delete-guard-nonlocal";
|
||||
let object = "object.bin";
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
|
||||
let marker = FileInfo {
|
||||
name: object.to_string(),
|
||||
version_id: Some(Uuid::parse_str("eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee").expect("version id should parse")),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(14).expect("marker timestamp should parse")),
|
||||
..Default::default()
|
||||
};
|
||||
let remote_dir = Uuid::parse_str("89898989-8989-8989-8989-898989898989").expect("remote data dir should parse");
|
||||
let mut remote = meta_regen_test_fileinfo(object, remote_dir, 15, 1);
|
||||
remote.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
remote.transition_tier = "WARM".to_string();
|
||||
remote.transitioned_objname = "remote/object.bin".to_string();
|
||||
|
||||
for metadata in [marker, remote] {
|
||||
let candidates = vec![metadata, FileInfo::default(), FileInfo::default(), FileInfo::default()];
|
||||
let errs = vec![
|
||||
None,
|
||||
Some(DiskError::FileNotFound),
|
||||
Some(DiskError::FileNotFound),
|
||||
Some(DiskError::FileNotFound),
|
||||
];
|
||||
assert_eq!(
|
||||
set.dangling_delete_safety(bucket, object, &candidates, &errs, &disks)
|
||||
.await
|
||||
.expect("non-local metadata should be classified"),
|
||||
DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_delete_guard_preserves_metadata_read_uncertainty() {
|
||||
let bucket = "bucket-delete-guard-read-error";
|
||||
let object = "object.bin";
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
|
||||
let metadata = vec![FileInfo::default(); disks.len()];
|
||||
|
||||
for read_error in [DiskError::Timeout, DiskError::DiskAccessDenied, DiskError::DiskNotFound] {
|
||||
let mut errs = vec![Some(DiskError::FileNotFound); disks.len()];
|
||||
errs[0] = Some(read_error);
|
||||
assert_eq!(
|
||||
set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
|
||||
.await
|
||||
.expect("metadata read uncertainty should be classified"),
|
||||
DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_no_parity_bitrot_reports_unrecoverable_integrity_failure() {
|
||||
let (dir, set) = formatted_single_disk_no_parity_set().await;
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::multipart_listing::paginate_multipart_listing;
|
||||
use futures::{StreamExt, stream};
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
@@ -36,7 +36,6 @@ const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
|
||||
#[cfg(test)]
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum MultipartCommitPause {
|
||||
PutPartBeforeLockAcquire,
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterRename,
|
||||
BeforeLockLost,
|
||||
@@ -48,10 +47,9 @@ struct MultipartCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
pause: MultipartCommitPause,
|
||||
expected_arrivals: usize,
|
||||
arrivals: AtomicUsize,
|
||||
armed: AtomicBool,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Semaphore,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -66,24 +64,13 @@ static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc
|
||||
#[cfg(test)]
|
||||
impl MultipartCommitBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
|
||||
Self::install_for_arrivals(bucket, object, pause, 1)
|
||||
}
|
||||
|
||||
pub(crate) fn install_for_arrivals(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
pause: MultipartCommitPause,
|
||||
expected_arrivals: usize,
|
||||
) -> Self {
|
||||
assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival");
|
||||
let state = Arc::new(MultipartCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
pause,
|
||||
expected_arrivals,
|
||||
arrivals: AtomicUsize::new(0),
|
||||
armed: AtomicBool::new(true),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Semaphore::new(0),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = MULTIPART_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -96,28 +83,20 @@ impl MultipartCommitBarrier {
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let arrived = self.state.arrived.notified();
|
||||
if self.state.arrivals.load(Ordering::Acquire) >= self.state.expected_arrivals {
|
||||
return;
|
||||
}
|
||||
arrived.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("multipart completion should reach the deterministic commit barrier");
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("multipart completion should reach the deterministic commit barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.add_permits(self.state.expected_arrivals);
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for MultipartCommitBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
self.state.release.notify_one();
|
||||
let mut slot = MULTIPART_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -138,20 +117,10 @@ async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartComm
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier
|
||||
&& let Ok(previous) = barrier.arrivals.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
|
||||
(current < barrier.expected_arrivals).then_some(current + 1)
|
||||
})
|
||||
&& barrier.armed.swap(false, Ordering::AcqRel)
|
||||
{
|
||||
let arrival = previous + 1;
|
||||
if arrival == barrier.expected_arrivals {
|
||||
barrier.arrived.notify_one();
|
||||
}
|
||||
barrier
|
||||
.release
|
||||
.acquire()
|
||||
.await
|
||||
.expect("multipart commit barrier should remain open")
|
||||
.forget();
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -724,8 +693,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockAcquire).await;
|
||||
// Serialize only the commit (rename_part), not the whole upload. Each
|
||||
// concurrent stream writes to its own unique temp dir (see `tmp_part`
|
||||
// above), so the encode/stream phase never conflicts and must stay
|
||||
@@ -3379,98 +3346,85 @@ mod tests {
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_holds_object_then_upload_lock_through_commit() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
|
||||
],
|
||||
async {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-layout-lock-order-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let create_opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
|
||||
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
|
||||
signaling.set_target(upload_resource.clone());
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-layout-lock-order-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let create_opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
|
||||
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
|
||||
signaling.set_target(upload_resource.clone());
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let observed = signaling.observed();
|
||||
let object_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &object_resource)
|
||||
.expect("completion should acquire the object lock");
|
||||
let upload_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &upload_resource)
|
||||
.expect("completion should acquire the upload lock");
|
||||
assert!(object_position < upload_position, "completion must acquire object before upload");
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
|
||||
|
||||
let list_store = set_disks.clone();
|
||||
let list_upload_id = upload_id.clone();
|
||||
let list = tokio::spawn(async move {
|
||||
list_store
|
||||
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(3).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
|
||||
|
||||
barrier.release();
|
||||
complete
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completion task should not panic")
|
||||
.expect("completion should commit after the barrier is released");
|
||||
let abort_err = abort
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let observed = signaling.observed();
|
||||
let object_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &object_resource)
|
||||
.expect("completion should acquire the object lock");
|
||||
let upload_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &upload_resource)
|
||||
.expect("completion should acquire the upload lock");
|
||||
assert!(object_position < upload_position, "completion must acquire object before upload");
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when abort acquires the lock");
|
||||
assert!(
|
||||
matches!(abort_err, StorageError::InvalidUploadID(..)),
|
||||
"abort should return InvalidUploadID after completion, got {abort_err:?}"
|
||||
);
|
||||
let list_err = list
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
|
||||
|
||||
let list_store = set_disks.clone();
|
||||
let list_upload_id = upload_id.clone();
|
||||
let list = tokio::spawn(async move {
|
||||
list_store
|
||||
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("ListParts task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
|
||||
assert!(
|
||||
matches!(list_err, StorageError::InvalidUploadID(..)),
|
||||
"ListParts should return InvalidUploadID after completion, got {list_err:?}"
|
||||
);
|
||||
},
|
||||
)
|
||||
});
|
||||
signaling.wait_for_attempts(3).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
|
||||
|
||||
barrier.release();
|
||||
complete
|
||||
.await
|
||||
.expect("completion task should not panic")
|
||||
.expect("completion should commit after the barrier is released");
|
||||
let abort_err = abort
|
||||
.await
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when abort acquires the lock");
|
||||
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||
let list_err = list
|
||||
.await
|
||||
.expect("ListParts task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
|
||||
assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
|
||||
@@ -25,10 +25,7 @@ use crate::set_disk::read::GetObjectDownstreamWriter;
|
||||
|
||||
use crate::bucket::lifecycle::{
|
||||
tier_delete_journal::{persist_tier_delete_journal_entry, remove_tier_delete_journal_entry},
|
||||
tier_sweeper::{
|
||||
Jentry, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_object_from_remote_tier_with_lease_idempotent,
|
||||
},
|
||||
tier_sweeper::{Jentry, RemoteTierDeleteOutcome, delete_object_from_remote_tier_with_lease_idempotent},
|
||||
transition_transaction::{
|
||||
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
|
||||
TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record,
|
||||
@@ -550,7 +547,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
&self.ctx.tier_config_mgr(),
|
||||
)
|
||||
.await?;
|
||||
return Ok(finish_set_disk_read_lock(gr, read_lock_guard.take(), None, bucket, object));
|
||||
return Ok(finish_set_disk_read_lock(
|
||||
gr,
|
||||
read_lock_guard.take(),
|
||||
lock_optimization_enabled,
|
||||
bucket,
|
||||
object,
|
||||
));
|
||||
}
|
||||
|
||||
// App-layer object data cache probe: metadata (etag/size) is resolved
|
||||
@@ -677,18 +680,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
return Ok(reader);
|
||||
}
|
||||
|
||||
let snapshot_lease = if lock_optimization_enabled {
|
||||
match fi.data_dir.filter(|data_dir| !data_dir.is_nil()) {
|
||||
Some(data_dir) => {
|
||||
let data_dir_path = format!("{object}/{data_dir}");
|
||||
acquire_snapshot_leases(&disks, bucket, &data_dir_path, fi.erasure.data_blocks).await
|
||||
}
|
||||
None => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
match codec_streaming_gate.decision {
|
||||
GetCodecStreamingDecision::Use => {
|
||||
match Self::get_object_decode_reader_with_fileinfo(
|
||||
@@ -717,7 +708,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
// Carry the hook probe result so the app layer skips its
|
||||
// now-redundant lookup on the streaming miss path (ODC-16).
|
||||
reader.body_source = body_source;
|
||||
return Ok(finish_set_disk_read_lock(reader, read_lock_guard.take(), snapshot_lease, bucket, object));
|
||||
return Ok(finish_set_disk_read_lock(
|
||||
reader,
|
||||
read_lock_guard.take(),
|
||||
lock_optimization_enabled,
|
||||
bucket,
|
||||
object,
|
||||
));
|
||||
}
|
||||
core::io_primitives::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
|
||||
record_get_codec_streaming_gate_decision(
|
||||
@@ -756,22 +753,15 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
let set_index = self.set_index;
|
||||
let pool_index = self.pool_index;
|
||||
let skip_verify = opts.skip_verify_bitrot;
|
||||
let producer_snapshot_lease = snapshot_lease.clone();
|
||||
if let Some(lease) = snapshot_lease {
|
||||
if lock_optimization_enabled {
|
||||
release_materialized_read_lock(&bucket, &object, read_lock_guard.take());
|
||||
reader.stream = Box::new(SnapshotLeaseReader {
|
||||
inner: reader.stream,
|
||||
lease: Some(lease),
|
||||
terminal_error: false,
|
||||
});
|
||||
debug!(bucket, object, "Lock optimization: replaced read lock with snapshot leases");
|
||||
debug!(bucket, object, "Lock optimization: released read lock before streaming read");
|
||||
}
|
||||
|
||||
// The producer shares the lease lifetime with the body so cancellation
|
||||
// cannot release the snapshot while the duplex task is still unwinding.
|
||||
// When lock optimization is disabled, keep the read-lock guard in the
|
||||
// task so it lives for the duration of the streaming read.
|
||||
tokio::spawn(async move {
|
||||
let _guard = read_lock_guard;
|
||||
let _snapshot_lease = producer_snapshot_lease;
|
||||
let mut writer = GetObjectDownstreamWriter::new(wd);
|
||||
// Do not wrap the entire read+write pipeline in `disk_read_timeout`.
|
||||
// `get_object_with_fileinfo` also waits on `writer`, so an outer timeout
|
||||
@@ -1673,11 +1663,7 @@ pub(crate) async fn cleanup_uncommitted_transition_upload(
|
||||
cleanup_version: &str,
|
||||
version_id_exact: bool,
|
||||
) -> std::io::Result<RemoteTierDeleteOutcome> {
|
||||
if version_id_exact {
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(object, cleanup_version, lease).await
|
||||
} else {
|
||||
delete_object_from_remote_tier_with_lease_idempotent(object, cleanup_version, lease, false).await
|
||||
}
|
||||
delete_object_from_remote_tier_with_lease_idempotent(object, cleanup_version, lease, version_id_exact).await
|
||||
}
|
||||
|
||||
fn log_transition_upload_cleanup_failure(lease: &TierOperationLease, object: &str, cleanup_version: &str, err: &std::io::Error) {
|
||||
@@ -1814,7 +1800,7 @@ impl Drop for TransitionUploadCleanup {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_rejected_transition_upload_durably(
|
||||
async fn cleanup_rejected_transition_upload_durably(
|
||||
lease: &TierOperationLease,
|
||||
object: &str,
|
||||
cleanup_version: &str,
|
||||
@@ -1827,13 +1813,6 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably(
|
||||
tier_name: lease.tier_name().to_string(),
|
||||
backend_identity: Some(lease.backend_identity()),
|
||||
version_id_exact,
|
||||
version_state: if !version_id_exact {
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled
|
||||
} else if cleanup_version == "null" {
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull
|
||||
} else {
|
||||
rustfs_filemeta::TransitionVersionState::Exact
|
||||
},
|
||||
};
|
||||
|
||||
let journal_error = if let Some(api) = api.as_ref() {
|
||||
@@ -1993,83 +1972,12 @@ async fn advance_and_save_transition_transaction(
|
||||
next: TransitionTransactionState,
|
||||
remote_version: Option<TransitionRemoteVersion>,
|
||||
) -> Result<()> {
|
||||
#[cfg(test)]
|
||||
record_transition_uploaded_save_attempt(transaction, next);
|
||||
transaction
|
||||
.advance(transaction.fence(), next, remote_version)
|
||||
.map_err(Error::other)?;
|
||||
save_transition_transaction_if_available(api, transaction).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct TransitionUploadedSaveProbeState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
attempts: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct TransitionUploadedSaveProbe {
|
||||
state: Arc<TransitionUploadedSaveProbeState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static TRANSITION_UPLOADED_SAVE_PROBE: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionUploadedSaveProbeState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl TransitionUploadedSaveProbe {
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(TransitionUploadedSaveProbeState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
attempts: std::sync::atomic::AtomicUsize::new(0),
|
||||
});
|
||||
let mut slot = TRANSITION_UPLOADED_SAVE_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition uploaded-save probe mutex should not poison");
|
||||
assert!(slot.is_none(), "transition uploaded-save probe must be installed by one test at a time");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
drop(slot);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
fn attempts(&self) -> usize {
|
||||
self.state.attempts.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for TransitionUploadedSaveProbe {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = TRANSITION_UPLOADED_SAVE_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition uploaded-save probe mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction, next: TransitionTransactionState) {
|
||||
if next != TransitionTransactionState::Uploaded {
|
||||
return;
|
||||
}
|
||||
let state = TRANSITION_UPLOADED_SAVE_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition uploaded-save probe mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|state| state.bucket == transaction.source.bucket && state.object == transaction.source.object)
|
||||
.cloned();
|
||||
if let Some(state) = state {
|
||||
state.attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_transition_transaction_if_available(api: Option<&Arc<ECStore>>, transaction_id: Uuid) -> Result<()> {
|
||||
if let Some(api) = api {
|
||||
return delete_transition_transaction_record(api.clone(), transaction_id).await;
|
||||
@@ -2320,78 +2228,11 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn persisted_transition_version(
|
||||
remote_version: &str,
|
||||
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
|
||||
persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn remote_version_state_writer_enabled() -> bool {
|
||||
remote_version_state_writer_fleet_proof().is_some()
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
|
||||
remote_version_state_writer_requested()
|
||||
.then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_requested() -> bool {
|
||||
remote_version_state_writer_enabled_for(
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
),
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
),
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof_matches(
|
||||
proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
|
||||
) -> bool {
|
||||
remote_version_state_writer_fleet_proof_matches_for(
|
||||
remote_version_state_writer_requested(),
|
||||
crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
|
||||
)
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
|
||||
requested && fleet_proof_matches
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
|
||||
requested && fleet_confirmed && fleet_proof_valid
|
||||
}
|
||||
|
||||
fn persisted_transition_version_with_gate(
|
||||
remote_version: &str,
|
||||
remote_version_state_writer_enabled: bool,
|
||||
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
|
||||
fn parse_transition_version_id(remote_version: &str) -> std::result::Result<Option<Uuid>, uuid::Error> {
|
||||
if remote_version.is_empty() {
|
||||
return Ok((None, rustfs_filemeta::TransitionVersionState::KnownDisabled));
|
||||
}
|
||||
|
||||
match Uuid::parse_str(remote_version) {
|
||||
Ok(version_id) if version_id.is_nil() => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier returned a nil object version ID",
|
||||
)),
|
||||
Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
|
||||
Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"opaque remote tier versions require the operator-attested live fleet capability gate",
|
||||
)),
|
||||
Err(_) if remote_version == "null" => {
|
||||
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
|
||||
}
|
||||
Err(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
|
||||
return Ok(None);
|
||||
}
|
||||
Uuid::parse_str(remote_version).map(|version_id| (!version_id.is_nil()).then_some(version_id))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2629,20 +2470,16 @@ mod transition_upload_completion_tests {
|
||||
|
||||
#[cfg(test)]
|
||||
mod transition_version_id_tests {
|
||||
use super::{
|
||||
TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
|
||||
remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
|
||||
};
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use super::{TransitionUploadCandidate, parse_transition_version_id};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn normalizes_persisted_unversioned_ids_and_preserves_put_constraints() {
|
||||
assert_eq!(parse_transition_version_id("").expect("empty remote version should be valid"), None);
|
||||
assert_eq!(
|
||||
persisted_transition_version("").expect("empty remote version identifies an unversioned tier"),
|
||||
(None, TransitionVersionState::KnownDisabled)
|
||||
parse_transition_version_id(&Uuid::nil().to_string()).expect("nil remote version should be valid"),
|
||||
None
|
||||
);
|
||||
assert!(persisted_transition_version(&Uuid::nil().to_string()).is_err());
|
||||
let nil_put_response = Uuid::nil().to_string();
|
||||
let nil_candidate = TransitionUploadCandidate::from_put_response(nil_put_response.clone());
|
||||
assert_eq!(nil_candidate.cleanup_version(), nil_put_response);
|
||||
@@ -2654,14 +2491,12 @@ mod transition_version_id_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_uuid_and_gates_opaque_remote_ids() {
|
||||
fn preserves_valid_remote_id_and_rejects_invalid_text() {
|
||||
let version_id = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
persisted_transition_version(&version_id.to_string()).expect("UUID remote version"),
|
||||
(Some(version_id.to_string()), TransitionVersionState::Exact)
|
||||
parse_transition_version_id(&version_id.to_string()).expect("UUID remote version should be valid"),
|
||||
Some(version_id)
|
||||
);
|
||||
assert!(persisted_transition_version("null").is_err());
|
||||
assert!(persisted_transition_version("opaque-version-token").is_err());
|
||||
assert_eq!(
|
||||
TransitionUploadCandidate::from_put_response(version_id.to_string()).cleanup_version(),
|
||||
version_id.to_string()
|
||||
@@ -2670,64 +2505,7 @@ mod transition_version_id_tests {
|
||||
TransitionUploadCandidate::from_put_response("opaque-version-token".to_string()).cleanup_version(),
|
||||
"opaque-version-token"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
|
||||
for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
|
||||
("old defaults", false, false, false, false),
|
||||
("missing fleet confirmation", true, false, true, false),
|
||||
("missing local opt-in", false, true, true, false),
|
||||
("missing fleet proof", true, true, false, false),
|
||||
("explicitly unconfirmed fleet", true, false, true, false),
|
||||
("rolled-back writer", false, true, true, false),
|
||||
("fully upgraded fleet", true, true, true, true),
|
||||
] {
|
||||
assert_eq!(
|
||||
remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
|
||||
expected,
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
|
||||
for (case, requested, fleet_proof_matches, expected) in [
|
||||
("operator gate closed", false, true, false),
|
||||
("fleet proof changed", true, false, false),
|
||||
("current authorization", true, true, true),
|
||||
] {
|
||||
assert_eq!(
|
||||
remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
|
||||
expected,
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fleet_gate_enables_null_and_opaque_remote_version_states() {
|
||||
for (remote_version, expected) in [
|
||||
("null", (Some("null".to_string()), TransitionVersionState::SuspendedNull)),
|
||||
(
|
||||
"opaque-version-token",
|
||||
(Some("opaque-version-token".to_string()), TransitionVersionState::Exact),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
persisted_transition_version_with_gate(remote_version, false).is_err(),
|
||||
"missing fleet confirmation must reject {remote_version:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
persisted_transition_version_with_gate(remote_version, true).expect("fleet-confirmed state must be persisted"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
persisted_transition_version_with_gate("", true).expect("empty remote version identifies an unversioned tier"),
|
||||
(None, TransitionVersionState::KnownDisabled)
|
||||
);
|
||||
assert!(parse_transition_version_id("not-a-uuid").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3887,16 +3665,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let dest_obj = transaction.remote_object.clone();
|
||||
let mut transition_meta = (*oi.user_defined).clone();
|
||||
transition_meta.insert("name".to_string(), object.to_string());
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
transaction.transaction_id.to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(transaction.backend_fingerprint),
|
||||
);
|
||||
|
||||
if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
|
||||
transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
|
||||
@@ -4007,24 +3775,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
|
||||
return Err(err.into());
|
||||
}
|
||||
let fleet_proof = remote_version_state_writer_fleet_proof();
|
||||
let remote_version_requires_fleet_proof =
|
||||
!candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
|
||||
let (transition_version_id, transition_version_state) =
|
||||
match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
|
||||
Ok(version) => version,
|
||||
Err(err) => {
|
||||
let cleanup_api = transition_cleanup_store(&self.ctx).await;
|
||||
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
|
||||
return Err(StorageError::Io(std::io::Error::other(format!(
|
||||
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
|
||||
))));
|
||||
}
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
if let Err(err) = advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
@@ -4042,6 +3792,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
|
||||
return Err(err);
|
||||
}
|
||||
let transition_version_id = match parse_transition_version_id(candidate.remote_version()) {
|
||||
Ok(version_id) => version_id,
|
||||
Err(err) => {
|
||||
if upload_cleanup.cleanup().await.is_ok() {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
}
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
|
||||
let mut commit_opts = opts.clone();
|
||||
commit_opts.no_lock = true;
|
||||
@@ -4099,11 +3859,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
current_fi.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
current_fi.transitioned_objname = dest_obj;
|
||||
current_fi.transition_tier = opts.transition.tier.clone();
|
||||
current_fi.transition_version_id = transition_version_id
|
||||
.as_deref()
|
||||
.and_then(|version_id| Uuid::parse_str(version_id).ok());
|
||||
current_fi.transition_version = transition_version_id;
|
||||
current_fi.transition_version_state = transition_version_state;
|
||||
current_fi.transition_version_id = transition_version_id;
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut current_fi.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
@@ -4140,21 +3896,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
|
||||
// This check is the fleet-proof lease linearization point. Revocation
|
||||
// blocks later commits; an already-authorized local quorum commit is
|
||||
// allowed to finish without holding a synchronous lock across I/O.
|
||||
if remote_version_requires_fleet_proof
|
||||
&& !fleet_proof
|
||||
.as_ref()
|
||||
.is_some_and(remote_version_state_writer_fleet_proof_matches)
|
||||
{
|
||||
drop(transition_lock_guard);
|
||||
if upload_cleanup.cleanup().await.is_ok() {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
}
|
||||
return Err(Error::other("remote version state fleet capability changed during transition"));
|
||||
}
|
||||
if let Err(err) = advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
@@ -4335,7 +4076,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let mut p_reader = PutObjReader::new(hash_reader);
|
||||
return match self_.clone().put_object(bucket, object, &mut p_reader, &ropts).await {
|
||||
Ok(restored_info) => {
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, &opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
@@ -4470,7 +4210,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return set_restore_header_fn(&mut oi, Some(err)).await;
|
||||
}
|
||||
};
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
@@ -4929,33 +4668,6 @@ mod transition_commit_failure_tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_unsupported_remote_versions_are_cleaned_up() {
|
||||
for remote_version in ["null", "opaque-version-token"] {
|
||||
let manager = TierConfigMgr::new();
|
||||
let backend = register_mock_tier(&manager, "WARM").await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&manager, "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
let candidate = TransitionUploadCandidate::from_put_response(remote_version.to_string());
|
||||
|
||||
persisted_transition_version(candidate.remote_version()).expect_err("unsupported writer version must fail closed");
|
||||
cleanup_rejected_transition_upload_durably(
|
||||
&lease,
|
||||
"remote/object",
|
||||
candidate.cleanup_version(),
|
||||
candidate.cleanup_version_is_exact(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("rejected remote upload must be cleaned up");
|
||||
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), candidate.cleanup_version().to_string())]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[serial_test::serial(restore_multipart_failure_point)]
|
||||
async fn multipart_restore_aborts_every_post_create_failure() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -6903,45 +6615,6 @@ mod transition_upload_integrity_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn unversioned_remote_version_is_persisted_without_version_id() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transition-unversioned-tier-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"unversioned remote tier must commit without a version id".repeat(1024);
|
||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
let save_probe = TransitionUploadedSaveProbe::install(bucket, object);
|
||||
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect("an unversioned remote version must commit");
|
||||
let (fi, _, _) = set_disks
|
||||
.get_object_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("committed unversioned transition metadata should be readable");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
assert_eq!(fi.transition_version_state, rustfs_filemeta::TransitionVersionState::KnownDisabled);
|
||||
assert_eq!(save_probe.attempts(), 1);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn opaque_remote_version_is_cleaned_before_parse_failure() {
|
||||
@@ -6957,7 +6630,7 @@ mod transition_upload_integrity_tests {
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect_err("an opaque remote version must fail closed until the capability gate is active");
|
||||
.expect_err("an unparseable remote version must fail closed");
|
||||
let removed_versions = backend.remove_versions().await;
|
||||
assert_eq!(removed_versions.len(), 1);
|
||||
assert_eq!(removed_versions[0].1, "opaque-version-token");
|
||||
@@ -6965,38 +6638,6 @@ mod transition_upload_integrity_tests {
|
||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn nil_remote_version_is_cleaned_exactly_before_transaction_persistence() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transition-nil-version-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"nil remote version must retain local data".repeat(1024);
|
||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let remote_version = Uuid::nil().to_string();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
let save_probe = TransitionUploadedSaveProbe::install(bucket, object);
|
||||
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect_err("a nil remote version must fail closed before transaction persistence");
|
||||
let put_versions = backend.put_versions().await;
|
||||
let removed_versions = backend.remove_versions().await;
|
||||
assert_eq!(removed_versions, put_versions);
|
||||
assert_eq!(removed_versions.len(), 1);
|
||||
assert_eq!(
|
||||
removed_versions.first().map(|(_, version)| version.as_str()),
|
||||
Some(remote_version.as_str())
|
||||
);
|
||||
assert_eq!(save_probe.attempts(), 0, "nil remote version must be rejected before saving Uploaded");
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn authoritative_read_failure_after_upload_cleans_exact_candidate_and_preserves_source() {
|
||||
@@ -7316,6 +6957,7 @@ mod transition_source_identity_matrix_tests {
|
||||
async fn transition_source_identity_field_matrix_rejects_single_field_drift() {
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum IdentityField {
|
||||
VersionId,
|
||||
DataDir,
|
||||
ModTime,
|
||||
Size,
|
||||
@@ -7331,6 +6973,7 @@ mod transition_source_identity_matrix_tests {
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
|
||||
for (index, field) in [
|
||||
IdentityField::VersionId,
|
||||
IdentityField::DataDir,
|
||||
IdentityField::ModTime,
|
||||
IdentityField::Size,
|
||||
@@ -7342,36 +6985,23 @@ mod transition_source_identity_matrix_tests {
|
||||
let object = format!("identity-{index}.bin");
|
||||
let payload = vec![u8::try_from(index + 1).expect("matrix index should fit u8"); 1024 * 1024];
|
||||
let mut reader = PutObjReader::from_vec(payload);
|
||||
let source_version_id = Uuid::new_v4();
|
||||
let source_opts = ObjectOptions {
|
||||
version_id: Some(source_version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let original = set_disks
|
||||
.put_object(bucket, &object, &mut reader, &source_opts)
|
||||
.put_object(bucket, &object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let (source, _, _) = set_disks
|
||||
.get_object_fileinfo(bucket, &object, &source_opts, true, false)
|
||||
.get_object_fileinfo(bucket, &object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("source metadata should resolve");
|
||||
assert_eq!(source.version_id, Some(source_version_id));
|
||||
assert_eq!(
|
||||
transition_source_identity(bucket, &object, &source, &source_opts, &get_raw_etag(&source.metadata))
|
||||
.expect("persisted versioned source identity should build")
|
||||
.version_mode,
|
||||
TransitionSourceVersionMode::Versioned
|
||||
);
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.clone(),
|
||||
etag: original.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
@@ -7384,6 +7014,7 @@ mod transition_source_identity_matrix_tests {
|
||||
|
||||
let mut changed = source.clone();
|
||||
match field {
|
||||
IdentityField::VersionId => changed.version_id = Some(Uuid::new_v4()),
|
||||
IdentityField::DataDir => changed.data_dir = Some(Uuid::new_v4()),
|
||||
IdentityField::ModTime => {
|
||||
changed.mod_time = changed.mod_time.map(|value| value + time::Duration::nanoseconds(1));
|
||||
@@ -7401,19 +7032,12 @@ mod transition_source_identity_matrix_tests {
|
||||
.await
|
||||
.expect("single-field metadata drift should be written");
|
||||
}
|
||||
let persisted_opts = ObjectOptions {
|
||||
version_id: changed.version_id.map(|version_id| version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (persisted, _, _) = set_disks
|
||||
.get_object_fileinfo(bucket, &object, &persisted_opts, true, false)
|
||||
.await
|
||||
.expect("drifted source metadata should resolve");
|
||||
put_barrier.release();
|
||||
|
||||
let result = transition.await.expect("transition task should not panic");
|
||||
assert!(result.is_err(), "transition must reject {field:?} drift");
|
||||
transition
|
||||
.await
|
||||
.expect("transition task should not panic")
|
||||
.expect_err("transition must reject a source whose identity changed after upload");
|
||||
let expected_attempts = index + 1;
|
||||
assert_eq!(backend.put_count().await, expected_attempts);
|
||||
assert_eq!(backend.remove_count().await, expected_attempts);
|
||||
@@ -7424,23 +7048,26 @@ mod transition_source_identity_matrix_tests {
|
||||
);
|
||||
|
||||
match field {
|
||||
IdentityField::DataDir => assert_ne!(source.data_dir, persisted.data_dir),
|
||||
IdentityField::ModTime => assert_ne!(source.mod_time, persisted.mod_time),
|
||||
IdentityField::Size => assert_ne!(source.size, persisted.size),
|
||||
IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&persisted.metadata)),
|
||||
IdentityField::VersionId => assert_ne!(source.version_id, changed.version_id),
|
||||
IdentityField::DataDir => assert_ne!(source.data_dir, changed.data_dir),
|
||||
IdentityField::ModTime => assert_ne!(source.mod_time, changed.mod_time),
|
||||
IdentityField::Size => assert_ne!(source.size, changed.size),
|
||||
IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&changed.metadata)),
|
||||
}
|
||||
if !matches!(field, IdentityField::VersionId) {
|
||||
assert_eq!(source.version_id, changed.version_id);
|
||||
}
|
||||
assert_eq!(source.version_id, persisted.version_id);
|
||||
if !matches!(field, IdentityField::DataDir) {
|
||||
assert_eq!(source.data_dir, persisted.data_dir);
|
||||
assert_eq!(source.data_dir, changed.data_dir);
|
||||
}
|
||||
if !matches!(field, IdentityField::ModTime) {
|
||||
assert_eq!(source.mod_time, persisted.mod_time);
|
||||
assert_eq!(source.mod_time, changed.mod_time);
|
||||
}
|
||||
if !matches!(field, IdentityField::Size) {
|
||||
assert_eq!(source.size, persisted.size);
|
||||
assert_eq!(source.size, changed.size);
|
||||
}
|
||||
if !matches!(field, IdentityField::Etag) {
|
||||
assert_eq!(get_raw_etag(&source.metadata), get_raw_etag(&persisted.metadata));
|
||||
assert_eq!(get_raw_etag(&source.metadata), get_raw_etag(&changed.metadata));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,10 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::lifecycle;
|
||||
use rustfs_filemeta::RestoreStatusOps;
|
||||
use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
|
||||
use s3s::dto::{RestoreStatus, Timestamp};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct RestoreCleanupIdentity {
|
||||
@@ -46,69 +43,6 @@ impl RestoreCleanupIdentity {
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn finalize_restore_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
obj_info: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let expected = RestoreCleanupIdentity::from_object_info(obj_info);
|
||||
let expected_operation_id = restore_operation_id_from_metadata(&opts.user_defined)?;
|
||||
let expected_etag = obj_info
|
||||
.etag
|
||||
.clone()
|
||||
.unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref()));
|
||||
let version_id = expected.version_id.map(|v| v.to_string());
|
||||
let _lock_guard = if !opts.no_lock {
|
||||
Some(
|
||||
self.acquire_write_lock_diag("restore_finalize_metadata", bucket, object)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let read_opts = ObjectOptions {
|
||||
version_id,
|
||||
versioned: opts.versioned,
|
||||
version_suspended: opts.version_suspended,
|
||||
..Default::default()
|
||||
};
|
||||
let (mut fi, _, disks) = self
|
||||
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
|
||||
.await?;
|
||||
if let Some(expected_operation_id) = expected_operation_id {
|
||||
require_restore_operation_id(&fi.metadata, expected_operation_id)?;
|
||||
}
|
||||
if !expected.matches_file_info(&fi, &expected_etag) {
|
||||
return Err(Error::other("restored object changed before restore metadata finalization"));
|
||||
}
|
||||
let restore_expiry =
|
||||
lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
|
||||
fi.metadata.insert(
|
||||
X_AMZ_RESTORE.as_str().to_string(),
|
||||
RestoreStatus {
|
||||
is_restore_in_progress: Some(false),
|
||||
restore_expiry_date: Some(Timestamp::from(restore_expiry)),
|
||||
}
|
||||
.to_string(),
|
||||
);
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
self.update_object_meta_with_opts(
|
||||
bucket,
|
||||
object,
|
||||
fi.clone(),
|
||||
disks.as_slice(),
|
||||
&UpdateMetadataOpts {
|
||||
replace_user_metadata: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
|
||||
}
|
||||
|
||||
pub async fn update_restore_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -13,11 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
|
||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions};
|
||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
async fn prime_metadata_generation(set_disks: &SetDisks, bucket: &str, object: &str) -> GetObjectMetadataCacheKey {
|
||||
@@ -84,57 +83,13 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
|
||||
let transitioned_generation = prime_metadata_generation(&set_disks, bucket, object).await;
|
||||
let mut restore_opts = ObjectOptions::default();
|
||||
restore_opts.transition.restore_request.days = Some(1);
|
||||
let restore_started = OffsetDateTime::now_utc();
|
||||
let expiry_from_restore_start = temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_ILM_DEBUG_DAY_SECS", Some("1")),
|
||||
("RUSTFS_ILM_PROCESS_TIME", Some("1")),
|
||||
],
|
||||
async {
|
||||
let expiry_from_restore_start = expected_expiry_time(restore_started, 1);
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
let restore_set = Arc::clone(&set_disks);
|
||||
let restore =
|
||||
tokio::spawn(async move { restore_set.restore_transitioned_object(bucket, object, &restore_opts).await });
|
||||
get_barrier.wait_until_paused().await;
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if expected_expiry_time(OffsetDateTime::now_utc(), 1) > expiry_from_restore_start {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("test clock should cross the next accelerated lifecycle boundary");
|
||||
get_barrier.release();
|
||||
restore
|
||||
.await
|
||||
.expect("restore task should join")
|
||||
.expect("restore should succeed");
|
||||
expiry_from_restore_start
|
||||
},
|
||||
)
|
||||
.await;
|
||||
Arc::clone(&set_disks)
|
||||
.restore_transitioned_object(bucket, object, &restore_opts)
|
||||
.await
|
||||
.expect("restore should succeed");
|
||||
assert_generation_reclaimed(&set_disks, &transitioned_generation).await;
|
||||
assert_eq!(backend.get_count().await, 1, "restore should read the remote candidate exactly once");
|
||||
|
||||
let restored_info = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("restored object metadata should be readable");
|
||||
let restore_status = parse_restore_obj_status(
|
||||
restored_info
|
||||
.user_defined
|
||||
.get(s3s::header::X_AMZ_RESTORE.as_str())
|
||||
.expect("completed restore header should be present"),
|
||||
)
|
||||
.expect("completed restore header should parse");
|
||||
assert!(
|
||||
restore_status.expiry().expect("completed restore should have an expiry") > expiry_from_restore_start,
|
||||
"restore expiry must be based on completion, not the time the remote copy started"
|
||||
);
|
||||
|
||||
let mut restored = Vec::new();
|
||||
set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
|
||||
@@ -87,7 +87,7 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
|
||||
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
|
||||
}
|
||||
|
||||
pub(crate) async fn await_bucket_namespace_operation<T, F>(
|
||||
async fn await_bucket_namespace_operation<T, F>(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &str,
|
||||
operation: &'static str,
|
||||
@@ -310,12 +310,13 @@ impl ECStore {
|
||||
meta.set_created(opts.created_at);
|
||||
|
||||
if opts.lock_enabled {
|
||||
meta.object_lock_config_xml = crate::bucket::utils::serialize::<ObjectLockConfiguration>(&enableObjcetLockConfig)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
|
||||
meta.object_lock_config_xml =
|
||||
crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
|
||||
if opts.versioning_enabled {
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
|
||||
await_bucket_namespace_operation(
|
||||
@@ -568,7 +569,6 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::error::StorageError;
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
@@ -590,7 +590,6 @@ mod tests {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Notify, OnceCell};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
@@ -1050,7 +1049,7 @@ mod tests {
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
for disk_path in &disk_paths {
|
||||
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
|
||||
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory"))
|
||||
.await
|
||||
.expect("empty directory remnant should be created");
|
||||
}
|
||||
@@ -1259,55 +1258,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_delete_preserves_put_committed_after_empty_scan() {
|
||||
let (_, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
|
||||
let object = "committed-after-empty-scan";
|
||||
let payload = b"object committed after DeleteBucket empty scan".to_vec();
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let barrier = install_delete_bucket_empty_scan_barrier();
|
||||
let delete_store = ecstore.clone();
|
||||
let delete_bucket = bucket.clone();
|
||||
let delete = tokio::spawn(async move {
|
||||
delete_store
|
||||
.delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
ecstore
|
||||
.put_object(&bucket, object, &mut put_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("PUT should commit while DeleteBucket is paused after its empty scan");
|
||||
|
||||
barrier.release();
|
||||
let err = delete
|
||||
.await
|
||||
.expect("DeleteBucket task should join")
|
||||
.expect_err("DeleteBucket must reject a PUT committed after its empty scan");
|
||||
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
|
||||
|
||||
let mut reader = ecstore
|
||||
.get_object_reader(&bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed object should remain readable after DeleteBucket fails");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("object body should remain readable");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_recreation_does_not_publish_unverified_usage() {
|
||||
|
||||
@@ -562,12 +562,10 @@ mod tests {
|
||||
},
|
||||
tier_sweeper::Jentry,
|
||||
transition_transaction::{
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
|
||||
TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
|
||||
TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator, load_transition_transaction_record,
|
||||
recover_transition_transaction_records, save_transition_transaction_record,
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
|
||||
TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
|
||||
TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
|
||||
save_transition_transaction_record,
|
||||
},
|
||||
},
|
||||
client::transition_api::ReaderImpl,
|
||||
@@ -577,7 +575,6 @@ mod tests {
|
||||
services::tier::{
|
||||
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
|
||||
tier::{TIER_CONFIG_FILE, TierConfigMgr},
|
||||
tier_config::{TierConfig, TierType, TierWasabi},
|
||||
tier_mutation_intent::{
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
|
||||
TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
|
||||
@@ -1166,37 +1163,6 @@ mod tests {
|
||||
.len()
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn register_transition_reconcile_test_tier(
|
||||
handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
tier_name: &str,
|
||||
) -> MockWarmBackend {
|
||||
let backend = MockWarmBackend::new();
|
||||
let mut manager = handle.write().await;
|
||||
manager.tiers.insert(
|
||||
tier_name.to_string(),
|
||||
TierConfig {
|
||||
version: "v1".to_string(),
|
||||
tier_type: TierType::Wasabi,
|
||||
name: tier_name.to_string(),
|
||||
wasabi: Some(TierWasabi {
|
||||
name: tier_name.to_string(),
|
||||
endpoint: "https://s3.wasabisys.com".to_string(),
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
bucket: "mock-tier".to_string(),
|
||||
prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
|
||||
region: "us-east-1".to_string(),
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
manager
|
||||
.install_test_driver(tier_name, Box::new(backend.clone()))
|
||||
.expect("mock fallback tier driver should install");
|
||||
backend
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn wait_for_tier_delete_journal_recovery(
|
||||
store: Arc<crate::store::ECStore>,
|
||||
@@ -1345,16 +1311,14 @@ mod tests {
|
||||
version_id: "version-a".to_string(),
|
||||
tier_name: tier_a.to_string(),
|
||||
backend_identity: Some(identity_a),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
let entry_b = Jentry {
|
||||
obj_name: "remote-b".to_string(),
|
||||
version_id: "version-b".to_string(),
|
||||
tier_name: tier_b.to_string(),
|
||||
backend_identity: Some(identity_b),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
let remove_a = backend_a.arm_failing_remove_barrier().await;
|
||||
persist_tier_delete_journal_entry(store_a.clone(), &entry_a)
|
||||
@@ -2756,12 +2720,10 @@ mod tests {
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transition_transaction_recovery_deletes_provider_recovered_unknown_upload() {
|
||||
let versioned_remote = uuid::Uuid::new_v4().to_string();
|
||||
let nil_remote = uuid::Uuid::nil().to_string();
|
||||
for (case, tier_name, remote_version) in [
|
||||
("missing", "TXPROBEMISSING", None),
|
||||
("unversioned", "TXPROBEUNVERSIONED", Some(String::new())),
|
||||
("versioned", "TXPROBEVERSIONED", Some(versioned_remote)),
|
||||
("nil-version", "TXPROBENILVERSION", Some(nil_remote)),
|
||||
] {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
|
||||
@@ -2772,7 +2734,7 @@ mod tests {
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
@@ -2843,157 +2805,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXOPERATOR";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("transaction should enter unknown upload outcome state");
|
||||
|
||||
let remote_version = uuid::Uuid::new_v4().to_string();
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
|
||||
backend
|
||||
.put(
|
||||
&transaction.remote_object,
|
||||
ReaderImpl::Body(candidate.clone()),
|
||||
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
|
||||
)
|
||||
.await
|
||||
.expect("mock backend should accept candidate");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
|
||||
let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("operator inspection should probe the candidate");
|
||||
assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
|
||||
|
||||
let wrong_version = uuid::Uuid::new_v4().to_string();
|
||||
let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
|
||||
.await
|
||||
.expect_err("a mismatched exact version must fail before deleting a candidate");
|
||||
assert!(matches!(
|
||||
err,
|
||||
TransitionOperatorError::CandidateVersionMismatch {
|
||||
expected,
|
||||
actual: TransitionOperatorProbe::VersionedPresent(ref observed),
|
||||
} if expected == wrong_version && observed == &remote_version
|
||||
));
|
||||
assert!(backend.contains(&transaction.remote_object).await);
|
||||
assert_eq!(backend.exact_remove_count(), 0);
|
||||
load_transition_transaction_record(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("an incorrect exact version must retain the transaction journal");
|
||||
|
||||
let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
|
||||
.await
|
||||
.expect("operator-confirmed exact candidate should be deleted");
|
||||
assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
|
||||
assert!(result.journal_observed_after_delete);
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
|
||||
load_transition_transaction_record(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("candidate deletion must retain the transaction journal");
|
||||
|
||||
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("a separately confirmed missing candidate should permit finalization");
|
||||
assert!(matches!(
|
||||
load_transition_transaction_record(store, transaction.transaction_id).await,
|
||||
Err(Error::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn operator_finalize_retains_record_without_missing_proof() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXOPERATORFAIL";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("transaction should enter unknown upload outcome state");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
|
||||
Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
|
||||
));
|
||||
load_transition_transaction_record(store, transaction.transaction_id)
|
||||
.await
|
||||
.expect("an unsupported probe must retain the transaction journal");
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -3004,7 +2815,7 @@ mod tests {
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXRESPONSELOSS";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let bucket = "transition-response-loss-bucket";
|
||||
let object = "source.bin";
|
||||
store
|
||||
|
||||
@@ -6895,7 +6895,6 @@ mod test {
|
||||
)
|
||||
.await
|
||||
.expect("local disk should be created");
|
||||
disk.make_volume("bucket").await.expect("test bucket should be created");
|
||||
let mut fi = FileInfo::new("object", 1, 1);
|
||||
fi.volume = "bucket".to_owned();
|
||||
fi.name = "object".to_owned();
|
||||
|
||||
@@ -141,7 +141,6 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
|
||||
const MAX_UPLOADS_LIST: usize = 10000;
|
||||
|
||||
mod bucket;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
mod heal;
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
@@ -428,11 +427,11 @@ impl ECStore {
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
static ref enableObjcetLockConfig: ObjectLockConfiguration = ObjectLockConfiguration {
|
||||
static ref ENABLED_OBJECT_LOCK_CONFIG: ObjectLockConfiguration = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
..Default::default()
|
||||
};
|
||||
static ref enableVersioningConfig: VersioningConfiguration = VersioningConfiguration {
|
||||
static ref ENABLED_VERSIONING_CONFIG: VersioningConfiguration = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
//! Test-only guard against tracing's process-global callsite-interest cache.
|
||||
//!
|
||||
//! Any test that installs its own subscriber and then asserts on span or event
|
||||
//! context is asserting on process-global state. See
|
||||
//! [`pin_callsite_interest_for_test`] for what goes wrong and why holding its
|
||||
//! guard fixes it.
|
||||
|
||||
/// Stop a sibling test thread from poisoning tracing's process-global callsite
|
||||
/// interest cache while this test asserts on span or event context.
|
||||
///
|
||||
/// `tracing` caches every callsite's `Interest` in **process-global** state, and
|
||||
/// the first thread to reach a callsite fixes that value. While at most one
|
||||
/// dispatcher is registered, tracing-core takes a fast path
|
||||
/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
|
||||
/// newly-registered callsite's interest from whichever subscriber is current
|
||||
/// *on the registering thread*, and registration happens exactly once (guarded
|
||||
/// by a CAS in `DefaultCallsite::register`).
|
||||
///
|
||||
/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
|
||||
/// reaches a production callsite first, from a thread with no subscriber
|
||||
/// installed: the interest is derived from that thread's `NoSubscriber` and
|
||||
/// cached as `Interest::never()` for the whole process. From then on the
|
||||
/// callsite is dead for *every* later caller, including a test that installed
|
||||
/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`, and
|
||||
/// a `warn!`/`info!` event never fires at all.
|
||||
///
|
||||
/// Registering a second, inert dispatcher closes both halves of that race:
|
||||
///
|
||||
/// * constructing it rebuilds every *already-registered* callsite's interest
|
||||
/// against the live dispatcher set — which includes the caller's subscriber —
|
||||
/// repairing whatever a sibling may already have poisoned; and
|
||||
/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
|
||||
/// a callsite registered *later* by any thread is resolved against that live
|
||||
/// set instead of the registering thread's `NoSubscriber`.
|
||||
///
|
||||
/// Call this **after** installing the test's subscriber, and hold the returned
|
||||
/// guard for the rest of the test. Only the `cargo test` fallback needs it:
|
||||
/// nextest runs each test in its own process, where there is no sibling thread
|
||||
/// to lose the race to (see `docs/testing/README.md`).
|
||||
#[must_use = "callsite interest is only pinned while the returned guard is held"]
|
||||
pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
|
||||
tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
|
||||
}
|
||||
@@ -219,16 +219,6 @@ impl ErasureInfo {
|
||||
}
|
||||
|
||||
// #[derive(Debug, Clone)]
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy, Default)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum TransitionVersionState {
|
||||
#[default]
|
||||
Unknown,
|
||||
KnownDisabled,
|
||||
SuspendedNull,
|
||||
Exact,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
|
||||
pub struct FileInfo {
|
||||
pub volume: String,
|
||||
@@ -240,10 +230,6 @@ pub struct FileInfo {
|
||||
pub transitioned_objname: String,
|
||||
pub transition_tier: String,
|
||||
pub transition_version_id: Option<Uuid>,
|
||||
#[serde(default)]
|
||||
pub transition_version: Option<String>,
|
||||
#[serde(default)]
|
||||
pub transition_version_state: TransitionVersionState,
|
||||
pub expire_restored: bool,
|
||||
pub data_dir: Option<Uuid>,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
@@ -473,10 +459,6 @@ impl FileInfo {
|
||||
if self.mod_time.is_none_or(|mod_time| mod_time <= OffsetDateTime::UNIX_EPOCH)
|
||||
|| (!allow_nil_version_id && self.version_id.is_some_and(|version_id| version_id.is_nil()))
|
||||
|| self.transition_version_id.is_some_and(|version_id| version_id.is_nil())
|
||||
|| self
|
||||
.transition_version
|
||||
.as_ref()
|
||||
.is_some_and(|version_id| version_id.is_empty())
|
||||
|| self.size != 0
|
||||
|| self.data_dir.is_some()
|
||||
|| self.mode.is_some()
|
||||
@@ -510,7 +492,6 @@ impl FileInfo {
|
||||
|| !self.transitioned_objname.is_empty()
|
||||
|| !self.transition_tier.is_empty()
|
||||
|| self.transition_version_id.is_some()
|
||||
|| self.transition_version.is_some()
|
||||
|| self.expire_restored
|
||||
|| self.size != 0
|
||||
|| self.data_dir.is_some()
|
||||
@@ -555,25 +536,6 @@ impl FileInfo {
|
||||
/// return `None`.
|
||||
pub fn validate(&self, mode: ValidationMode) -> Result<Option<ValidatedErasureLayout>> {
|
||||
self.validate_collection_bounds()?;
|
||||
if let (Some(version), Some(version_id)) = (&self.transition_version, self.transition_version_id)
|
||||
&& Uuid::parse_str(version).ok() != Some(version_id)
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let transition_state_valid = match self.transition_version_state {
|
||||
TransitionVersionState::Unknown => true,
|
||||
TransitionVersionState::KnownDisabled => self.transition_version.is_none() && self.transition_version_id.is_none(),
|
||||
TransitionVersionState::SuspendedNull => {
|
||||
self.transition_version.as_deref() == Some("null") && self.transition_version_id.is_none()
|
||||
}
|
||||
TransitionVersionState::Exact => self
|
||||
.transition_version
|
||||
.as_deref()
|
||||
.is_some_and(|version| version != "null" && !version.is_empty()),
|
||||
};
|
||||
if !transition_state_valid {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
|
||||
let erasure_layout = match mode {
|
||||
ValidationMode::RequireErasure => Some(self.validate_erasure_geometry()?),
|
||||
@@ -870,8 +832,6 @@ impl FileInfo {
|
||||
&& self.transition_tier == other.transition_tier
|
||||
&& self.transitioned_objname == other.transitioned_objname
|
||||
&& self.transition_version_id == other.transition_version_id
|
||||
&& self.transition_version == other.transition_version
|
||||
&& self.transition_version_state == other.transition_version_state
|
||||
}
|
||||
|
||||
/// Check if metadata maps are equal
|
||||
@@ -1391,15 +1351,6 @@ mod tests {
|
||||
assert_file_corrupt(&fi, ValidationMode::DeleteOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_read_validation_rejects_conflicting_transition_versions() {
|
||||
let mut fi = one_shard_validation_fileinfo(1);
|
||||
fi.transition_version_id = Some(Uuid::new_v4());
|
||||
fi.transition_version = Some(Uuid::new_v4().to_string());
|
||||
|
||||
assert_file_corrupt(&fi, ValidationMode::RequireErasure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_read_validation_requires_canonical_delete_marker_shape() {
|
||||
let marker = FileInfo {
|
||||
@@ -1771,12 +1722,6 @@ mod tests {
|
||||
transitioned_objname,
|
||||
transition_tier,
|
||||
transition_version_id,
|
||||
transition_version: transition_version_id.map(|version_id| version_id.to_string()),
|
||||
transition_version_state: if transition_version_id.is_some() {
|
||||
TransitionVersionState::Exact
|
||||
} else {
|
||||
TransitionVersionState::Unknown
|
||||
},
|
||||
expire_restored,
|
||||
data_dir,
|
||||
mod_time,
|
||||
|
||||
@@ -356,14 +356,6 @@ impl FileMeta {
|
||||
.versions
|
||||
.partition_point(|existing| existing.header.sorts_before(&new_shallow.header));
|
||||
self.versions.insert(insert_pos, new_shallow);
|
||||
// `partition_point` only returns the canonical slot when `versions` is
|
||||
// already canonically ordered, and nothing establishes that: `FileMeta::load`
|
||||
// replays the on-disk order verbatim, and metadata written before the
|
||||
// canonical-order fix ordered equal-`mod_time` ties by insertion rather than
|
||||
// by `sorts_before`. Re-sort so an insert leaves the list canonical either
|
||||
// way, matching what `set_idx` already does on the replace path. Cheap: after
|
||||
// a correctly placed insert the slice is a single sorted run.
|
||||
self.sort_by_mod_time();
|
||||
Ok(())
|
||||
|
||||
// if !ver.valid() {
|
||||
@@ -1223,57 +1215,6 @@ mod test {
|
||||
assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header));
|
||||
}
|
||||
|
||||
/// `add_version_filemata` positions an inserted version with
|
||||
/// `partition_point(sorts_before)`, which only yields the canonical slot when
|
||||
/// `versions` is already canonically ordered. Nothing establishes that
|
||||
/// precondition on load: `FileMeta::unmarshal_msg` pushes versions in file
|
||||
/// order, and metadata written before the canonical-order fix ordered
|
||||
/// equal-`mod_time` ties by insertion instead of by `sorts_before`. An insert
|
||||
/// into such a list must still leave it canonical, the same way `set_idx`
|
||||
/// already re-sorts on the replace path.
|
||||
#[test]
|
||||
fn add_version_filemata_canonicalizes_versions_loaded_out_of_order() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
let first = version_for_ordering(VersionType::Object, Uuid::from_u128(70), mod_time, 70);
|
||||
let second = version_for_ordering(VersionType::Object, Uuid::from_u128(80), mod_time, 80);
|
||||
let (early, late) = if first.header().sorts_before(&second.header()) {
|
||||
(first, second)
|
||||
} else {
|
||||
(second, first)
|
||||
};
|
||||
|
||||
// Persist the pair the wrong way round to emulate a pre-canonical-order
|
||||
// xl.meta, bypassing add_version_filemata so the bad order really lands on
|
||||
// the wire.
|
||||
let mut legacy = FileMeta::new();
|
||||
legacy
|
||||
.versions
|
||||
.push(FileMetaShallowVersion::try_from(late).expect("shallow later version"));
|
||||
legacy
|
||||
.versions
|
||||
.push(FileMetaShallowVersion::try_from(early).expect("shallow earlier version"));
|
||||
|
||||
let mut loaded =
|
||||
FileMeta::load(&legacy.marshal_msg().expect("serialize legacy metadata")).expect("reload legacy metadata");
|
||||
assert!(
|
||||
!loaded.versions[0].header.sorts_before(&loaded.versions[1].header),
|
||||
"fixture must reach add_version_filemata non-canonically ordered"
|
||||
);
|
||||
|
||||
loaded
|
||||
.add_version_filemata(version_for_ordering(VersionType::Delete, Uuid::from_u128(90), mod_time, 90))
|
||||
.expect("insert equal-time delete marker");
|
||||
|
||||
assert_eq!(loaded.versions.len(), 3);
|
||||
assert!(
|
||||
loaded
|
||||
.versions
|
||||
.windows(2)
|
||||
.all(|pair| pair[0].header.sorts_before(&pair[1].header)),
|
||||
"insert must leave the version list canonically ordered"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression for backlog#799 B16: replication reset state must be
|
||||
/// persisted under both internal prefixes, never as a bare ARN. A bare-ARN
|
||||
/// key (produced by `ObjectInfo::replication_state`) has no internal prefix,
|
||||
|
||||
@@ -26,14 +26,13 @@ use super::msgp_decode::{
|
||||
PrependByteReader, prealloc_hint, read_exact_vec, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value,
|
||||
};
|
||||
use super::*;
|
||||
use crate::{ChecksumInfo, TransitionVersionState};
|
||||
use crate::ChecksumInfo;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use rustfs_utils::http::{
|
||||
RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID,
|
||||
SUFFIX_TIER_FV_MARKER, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
|
||||
get_bytes, get_consistent_bytes, get_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes,
|
||||
strip_internal_prefix,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, contains_key_bytes, get_bytes, get_consistent_bytes, get_str,
|
||||
has_internal_suffix, insert_bytes, is_internal_key, remove_bytes, strip_internal_prefix,
|
||||
};
|
||||
|
||||
const MSGPACK_EXT8: u8 = 0xc7;
|
||||
@@ -44,7 +43,6 @@ const MSGPACK_FIXEXT8: u8 = 0xd7;
|
||||
const MSGPACK_TIME_EXT_LEGACY: i8 = 5;
|
||||
const MSGPACK_TIME_EXT_OFFICIAL: i8 = -1;
|
||||
const MSGPACK_TIME_LEN: u8 = 12;
|
||||
const MAX_TRANSITION_VERSION_LEN: usize = 1024;
|
||||
|
||||
/// Sentinel signature returned when a version has no computable body (invalid /
|
||||
/// missing inner object). Mirrors MinIO's `signatureErr` so such versions never
|
||||
@@ -253,93 +251,23 @@ fn parse_legacy_uuid_bytes(bytes: &[u8], field: &str) -> Result<Option<Uuid>> {
|
||||
|
||||
/// Decode a stored transitioned-version-id from a version's `meta_sys`.
|
||||
///
|
||||
/// Legacy RustFS writes used 16 raw UUID bytes. New writes and MinIO-migrated
|
||||
/// records use the provider's exact UTF-8 version text. Empty, nil UUID, and
|
||||
/// malformed bytes are not usable remote versions.
|
||||
fn transitioned_version_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Result<Option<String>> {
|
||||
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(value) = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value = value.to_vec();
|
||||
/// RustFS writes it as 16 raw UUID bytes; MinIO-migrated tiered objects store
|
||||
/// the remote tier's version id as a UUID *string*. Accept both, and treat any
|
||||
/// absent / nil / otherwise-unparseable value as "no tier version" (matching the
|
||||
/// tolerant pre-hardening behavior) rather than failing the whole object read —
|
||||
/// a malformed tier id must not make an otherwise-readable object unreadable.
|
||||
fn transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
|
||||
let value = get_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)?;
|
||||
if value.is_empty() {
|
||||
return Ok(None);
|
||||
return None;
|
||||
}
|
||||
if let Ok(id) = Uuid::from_slice(&value) {
|
||||
return Ok((!id.is_nil()).then(|| id.to_string()));
|
||||
return (!id.is_nil()).then_some(id);
|
||||
}
|
||||
let Ok(value) = String::from_utf8(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if value.is_empty()
|
||||
|| value.len() > MAX_TRANSITION_VERSION_LEN
|
||||
|| value.chars().any(char::is_control)
|
||||
|| Uuid::parse_str(&value).is_ok_and(|id| id.is_nil())
|
||||
{
|
||||
Ok(None)
|
||||
} else {
|
||||
Ok(Some(value))
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_version_state_from_meta_sys(
|
||||
meta_sys: &HashMap<String, Vec<u8>>,
|
||||
version: Option<&str>,
|
||||
) -> Result<TransitionVersionState> {
|
||||
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
|
||||
return Ok(TransitionVersionState::Unknown);
|
||||
}
|
||||
let value = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or(Error::FileCorrupt)?;
|
||||
let state = match value {
|
||||
b"known-disabled" => TransitionVersionState::KnownDisabled,
|
||||
b"suspended-null" => TransitionVersionState::SuspendedNull,
|
||||
b"exact" => TransitionVersionState::Exact,
|
||||
b"unknown" => TransitionVersionState::Unknown,
|
||||
_ => return Err(Error::FileCorrupt),
|
||||
};
|
||||
let valid = match state {
|
||||
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
|
||||
TransitionVersionState::SuspendedNull => version == Some("null"),
|
||||
TransitionVersionState::Exact => version.is_some_and(|value| value != "null"),
|
||||
};
|
||||
valid.then_some(state).ok_or(Error::FileCorrupt)
|
||||
}
|
||||
|
||||
fn transition_version_state_bytes(state: TransitionVersionState) -> &'static [u8] {
|
||||
match state {
|
||||
TransitionVersionState::Unknown => b"unknown",
|
||||
TransitionVersionState::KnownDisabled => b"known-disabled",
|
||||
TransitionVersionState::SuspendedNull => b"suspended-null",
|
||||
TransitionVersionState::Exact => b"exact",
|
||||
}
|
||||
}
|
||||
|
||||
fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: TransitionVersionState) {
|
||||
if state == TransitionVersionState::Unknown {
|
||||
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
|
||||
} else {
|
||||
insert_bytes(
|
||||
meta_sys,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
transition_version_state_bytes(state).to_vec(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn legacy_transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
|
||||
transitioned_version_from_meta_sys(meta_sys)
|
||||
std::str::from_utf8(&value)
|
||||
.ok()
|
||||
.flatten()
|
||||
.and_then(|value| Uuid::parse_str(&value).ok())
|
||||
}
|
||||
|
||||
fn transitioned_version_bytes(fi: &FileInfo) -> Option<Vec<u8>> {
|
||||
fi.transition_version
|
||||
.as_ref()
|
||||
.map(|version| version.as_bytes().to_vec())
|
||||
.or_else(|| fi.transition_version_id.map(|version_id| version_id.as_bytes().to_vec()))
|
||||
.and_then(|s| Uuid::parse_str(s.trim()).ok())
|
||||
.filter(|id| !id.is_nil())
|
||||
}
|
||||
|
||||
fn parse_legacy_erasure_algo(value: &str) -> ErasureAlgo {
|
||||
@@ -2470,9 +2398,7 @@ impl MetaObject {
|
||||
let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
|
||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
.unwrap_or_default();
|
||||
let transition_version = transitioned_version_from_meta_sys(&self.meta_sys)?;
|
||||
let transition_version_state = transition_version_state_from_meta_sys(&self.meta_sys, transition_version.as_deref())?;
|
||||
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
||||
let transition_version_id = transitioned_version_id_from_meta_sys(&self.meta_sys);
|
||||
let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
|
||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
.unwrap_or_default();
|
||||
@@ -2493,8 +2419,6 @@ impl MetaObject {
|
||||
transition_status,
|
||||
transitioned_objname,
|
||||
transition_version_id,
|
||||
transition_version,
|
||||
transition_version_state,
|
||||
transition_tier,
|
||||
..Default::default()
|
||||
})
|
||||
@@ -2507,12 +2431,13 @@ impl MetaObject {
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
fi.transitioned_objname.as_bytes().to_vec(),
|
||||
);
|
||||
if let Some(transition_version) = transitioned_version_bytes(fi) {
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
} else {
|
||||
remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
|
||||
if let Some(transition_version_id) = fi.transition_version_id.as_ref() {
|
||||
insert_bytes(
|
||||
&mut self.meta_sys,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
transition_version_id.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state);
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
|
||||
if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes());
|
||||
@@ -2576,7 +2501,6 @@ impl MetaObject {
|
||||
SUFFIX_TRANSITION_TIER,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
] {
|
||||
if let Some(v) = get_bytes(&self.meta_sys, suffix) {
|
||||
insert_bytes(&mut delete_marker.meta_sys, suffix, v);
|
||||
@@ -2638,11 +2562,8 @@ impl From<FileInfo> for MetaObject {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(transition_version) = transitioned_version_bytes(&value) {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
if let Some(vid) = &value.transition_version_id {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, vid.as_bytes().to_vec());
|
||||
}
|
||||
|
||||
if !value.transition_tier.is_empty() {
|
||||
@@ -2785,11 +2706,7 @@ impl MetaDeleteMarker {
|
||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
fi.transition_version = transitioned_version_from_meta_sys(&self.meta_sys).ok().flatten();
|
||||
fi.transition_version_id = legacy_transitioned_version_id_from_meta_sys(&self.meta_sys);
|
||||
fi.transition_version_state =
|
||||
transition_version_state_from_meta_sys(&self.meta_sys, fi.transition_version.as_deref())
|
||||
.unwrap_or(TransitionVersionState::Unknown);
|
||||
fi.transition_version_id = transitioned_version_id_from_meta_sys(&self.meta_sys);
|
||||
}
|
||||
|
||||
fi
|
||||
@@ -2942,11 +2859,8 @@ impl From<FileInfo> for MetaDeleteMarker {
|
||||
value.transitioned_objname.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
if let Some(transition_version) = transitioned_version_bytes(&value) {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() || value.tier_free_version() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
if let Some(version_id) = value.transition_version_id {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, version_id.as_bytes().to_vec());
|
||||
}
|
||||
if !value.transition_tier.is_empty() {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
|
||||
@@ -3498,7 +3412,7 @@ mod tests {
|
||||
.insert("x-rustfs-internal-healing".to_string(), "true".to_string());
|
||||
marker.metadata.insert("content-type".to_string(), "text/plain".to_string());
|
||||
let remote_version_id = Uuid::new_v4();
|
||||
marker.transition_version = Some(remote_version_id.to_string());
|
||||
marker.transition_version_id = Some(remote_version_id);
|
||||
|
||||
let converted = MetaDeleteMarker::from(marker);
|
||||
|
||||
@@ -3506,19 +3420,7 @@ mod tests {
|
||||
assert_eq!(converted.meta_sys.get("x-minio-internal-purgestatus"), Some(&b"pending".to_vec()));
|
||||
assert_eq!(
|
||||
get_bytes(&converted.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID),
|
||||
Some(remote_version_id.to_string().into_bytes())
|
||||
);
|
||||
assert_eq!(
|
||||
converted
|
||||
.meta_sys
|
||||
.get(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}")),
|
||||
Some(&remote_version_id.to_string().into_bytes())
|
||||
);
|
||||
assert_eq!(
|
||||
converted
|
||||
.meta_sys
|
||||
.get(&format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)),
|
||||
Some(&remote_version_id.to_string().into_bytes())
|
||||
Some(remote_version_id.as_bytes().to_vec())
|
||||
);
|
||||
assert!(!converted.meta_sys.contains_key("x-rustfs-internal-healing"));
|
||||
assert!(!converted.meta_sys.contains_key("content-type"));
|
||||
@@ -4195,129 +4097,19 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("into_fileinfo");
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_id_opaque_text_is_preserved() {
|
||||
fn meta_object_transition_version_id_unparseable_stays_readable_as_none() {
|
||||
// A non-UUID / non-16-byte tier version id must NOT make the object
|
||||
// unreadable; it is tolerated as "no tier version" (compat with
|
||||
// pre-hardening behavior and foreign/edge metadata).
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"opaque-generation-42".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"not-a-uuid".to_vec());
|
||||
let fi = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("opaque transition version id must decode");
|
||||
.expect("unparseable transition version id must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_exact_round_trips_dual_keys() {
|
||||
let id = sample_version_id();
|
||||
let expected_version = id.to_string();
|
||||
let fi = FileInfo {
|
||||
transition_status: "complete".to_string(),
|
||||
transition_version: Some(expected_version.clone()),
|
||||
transition_version_state: TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let object = MetaObject::from(fi);
|
||||
assert_eq!(
|
||||
object
|
||||
.meta_sys
|
||||
.get(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"))
|
||||
.map(Vec::as_slice),
|
||||
Some(b"exact".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
object
|
||||
.meta_sys
|
||||
.get(&format!(
|
||||
"{}{SUFFIX_TRANSITIONED_VERSION_STATE}",
|
||||
rustfs_utils::http::MINIO_INTERNAL_PREFIX
|
||||
))
|
||||
.map(Vec::as_slice),
|
||||
Some(b"exact".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
legacy_transitioned_version_id_from_meta_sys(&object.meta_sys),
|
||||
Some(id),
|
||||
"UUID exact writes must remain readable by the legacy UUID consumer"
|
||||
);
|
||||
let decoded = object.into_fileinfo("b", "k", false).expect("exact state should round trip");
|
||||
assert_eq!(decoded.transition_version_state, TransitionVersionState::Exact);
|
||||
assert_eq!(decoded.transition_version.as_deref(), Some(expected_version.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_transition_known_disabled_removes_stale_version_dual_keys() {
|
||||
let mut meta_sys = HashMap::new();
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, b"stale-legacy-version".to_vec());
|
||||
let mut object = make_meta_object_with_sys(meta_sys);
|
||||
object.set_transition(&FileInfo {
|
||||
transition_status: TRANSITION_COMPLETE.to_string(),
|
||||
transitioned_objname: "remote/object".to_string(),
|
||||
transition_version_state: TransitionVersionState::KnownDisabled,
|
||||
transition_tier: "WARM".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(get_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID), None);
|
||||
assert!(
|
||||
!object
|
||||
.meta_sys
|
||||
.contains_key(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}"))
|
||||
);
|
||||
assert!(
|
||||
!object
|
||||
.meta_sys
|
||||
.contains_key(&format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX))
|
||||
);
|
||||
let decoded = object
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("known-disabled transition must remain readable after replacing stale metadata");
|
||||
assert_eq!(decoded.transition_version, None);
|
||||
assert_eq!(decoded.transition_version_state, TransitionVersionState::KnownDisabled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_conflict_fails_closed() {
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, sample_version_id().as_bytes().to_vec());
|
||||
sys.insert(format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"), b"exact".to_vec());
|
||||
sys.insert(
|
||||
format!("{}{SUFFIX_TRANSITIONED_VERSION_STATE}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
|
||||
b"known-disabled".to_vec(),
|
||||
);
|
||||
|
||||
make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect_err("conflicting state keys must fail closed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_id_invalid_utf8_yields_none() {
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, vec![0xff]);
|
||||
let fi = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("invalid transition version bytes must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_id_unsafe_text_yields_none() {
|
||||
for value in [b"opaque\0version".to_vec(), vec![b'x'; MAX_TRANSITION_VERSION_LEN + 1]] {
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, value);
|
||||
let fi = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("unsafe transition version text must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4331,7 +4123,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("string-form transition version id must decode");
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4361,14 +4152,16 @@ mod tests {
|
||||
}
|
||||
.into_fileinfo("b", "k", false);
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_free_version_transition_version_id_opaque_text_is_preserved() {
|
||||
fn delete_marker_free_version_transition_version_id_unparseable_stays_readable() {
|
||||
// A malformed tier version id must not make a free-version record corrupt:
|
||||
// it decodes to None and stays readable. Otherwise free-version expiry
|
||||
// fails and the remote-tier object leaks.
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"opaque-generation-42".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"not-a-uuid".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITION_TIER, b"WARM".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_OBJECTNAME, b"remote-object".to_vec());
|
||||
let fi = MetaDeleteMarker {
|
||||
@@ -4379,9 +4172,8 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false);
|
||||
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
|
||||
fi.validate_for_metadata_read()
|
||||
.expect("free-version record with an opaque tier id must remain readable");
|
||||
.expect("free-version record with an unparseable tier id must remain readable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4401,7 +4193,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false);
|
||||
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -26,9 +26,6 @@ documentation = "https://docs.rs/rustfs-heal/latest/rustfs_heal/"
|
||||
keywords = ["RustFS", "heal", "erasure-coding", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "filesystem"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true }
|
||||
rustfs-concurrency = { workspace = true }
|
||||
@@ -56,7 +53,7 @@ serial_test = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
walkdir = { workspace = true }
|
||||
http = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
temp-env = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -482,7 +482,7 @@ impl HealChannelProcessor {
|
||||
request_id: client_token,
|
||||
success: false,
|
||||
data: None,
|
||||
error: Some(error_text),
|
||||
error: Some(error_text.clone()),
|
||||
};
|
||||
let _ = response_tx.send(Ok(response.clone()));
|
||||
self.publish_response(response);
|
||||
|
||||
@@ -576,7 +576,7 @@ mod tests {
|
||||
assert_eq!(handler.event_count(), 1);
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
handler.add_event(event.clone());
|
||||
assert_eq!(handler.event_count(), 3);
|
||||
}
|
||||
|
||||
@@ -593,7 +593,7 @@ mod tests {
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event); // Should remove oldest
|
||||
handler.add_event(event.clone()); // Should remove oldest
|
||||
|
||||
assert_eq!(handler.event_count(), 2);
|
||||
}
|
||||
@@ -610,7 +610,7 @@ mod tests {
|
||||
};
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
handler.add_event(event.clone());
|
||||
|
||||
let events = handler.get_events();
|
||||
assert_eq!(events.len(), 2);
|
||||
|
||||
@@ -2892,7 +2892,7 @@ fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTa
|
||||
gauge!(
|
||||
"rustfs_heal_task_running",
|
||||
"type" => type_label.to_string(),
|
||||
"set" => set_label
|
||||
"set" => set_label.clone()
|
||||
)
|
||||
.set(count as f64);
|
||||
}
|
||||
@@ -3482,7 +3482,7 @@ mod tests {
|
||||
);
|
||||
|
||||
assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(runnable), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(runnable.clone()), QueuePushOutcome::Accepted);
|
||||
|
||||
let mut running = HashMap::new();
|
||||
running.insert("pool_0_set_1".to_string(), 1);
|
||||
|
||||
@@ -28,7 +28,6 @@ use rustfs_heal::heal::storage::{
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
future::Future,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
@@ -53,9 +52,15 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Disable the grace window while the destructive heal decision is evaluated.
|
||||
async fn with_dangling_grace_disabled<T>(future: impl Future<Output = T>) -> T {
|
||||
temp_env::async_with_vars([(GRACE_ENV, Some("0"))], future).await
|
||||
/// Disable the dangling-delete grace window so the destructive path is genuinely
|
||||
/// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD
|
||||
/// be dangling-deleted here. With grace at its 1h default the delete path would be
|
||||
/// masked and the test would prove nothing.
|
||||
fn disable_dangling_grace() {
|
||||
// Safe under nextest: each test runs in its own process and is `#[serial]`.
|
||||
unsafe {
|
||||
std::env::set_var(GRACE_ENV, "0");
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the
|
||||
@@ -245,6 +250,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn union_meta_lost_data_present_is_repaired_not_destroyed() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
|
||||
let bucket = "b920-meta-lost";
|
||||
let object = "obj.bin";
|
||||
@@ -262,10 +268,10 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
// Heal the version through the real heal storage (Deep).
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
assert!(
|
||||
error.is_none(),
|
||||
"recoverable version must heal without error (must NOT be dangling-deleted): {error:?}"
|
||||
@@ -294,6 +300,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-torn";
|
||||
let object = "obj.bin";
|
||||
@@ -311,10 +318,10 @@ mod serial_tests {
|
||||
// is a real destructive action, not a no-op).
|
||||
assert_eq!(count_part_files(&object_dir(&disk_paths[0], bucket, object)), 1);
|
||||
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
// A dangling delete reports the version as gone (FileVersionNotFound),
|
||||
// proving the destructive path fired for a genuinely torn write.
|
||||
assert!(error.is_some(), "a torn (< data_blocks) version must NOT be silently treated as healed");
|
||||
@@ -342,6 +349,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
|
||||
let bucket = "b920-reconstruct";
|
||||
let object = "obj.bin";
|
||||
@@ -365,10 +373,10 @@ mod serial_tests {
|
||||
"disk-walk must enumerate the reconstructable sub-quorum version"
|
||||
);
|
||||
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
assert!(error.is_none(), "reconstructable version must heal cleanly: {error:?}");
|
||||
|
||||
// part.* + xl.meta physically restored on the 4 wiped disks.
|
||||
@@ -455,6 +463,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn offline_disk_during_walk_does_not_dangling_delete() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-offline";
|
||||
let object = "obj.bin";
|
||||
@@ -482,7 +491,8 @@ mod serial_tests {
|
||||
remove: false,
|
||||
..Default::default()
|
||||
};
|
||||
let (_result, error) = with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &normal_opts))
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &normal_opts)
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
assert!(error.is_none(), "quorum-present object must not be destroyed: {error:?}");
|
||||
@@ -500,6 +510,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b1044-deep-verify";
|
||||
let object = "obj.bin";
|
||||
@@ -513,10 +524,10 @@ mod serial_tests {
|
||||
assert_eq!(count_part_files(&object_dir(disk, bucket, object)), 1, "intact shard must remain present");
|
||||
}
|
||||
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("deep heal_object call must not itself error");
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("deep heal_object call must not itself error");
|
||||
assert!(error.is_none(), "Deep heal must retain the three intact EC2+2 shards: {error:?}");
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -390,10 +390,8 @@ mod serial_tests {
|
||||
|
||||
// ─── 1️⃣ delete format.json on one disk ──────────────
|
||||
let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
|
||||
let failed_disk_path = disk_paths[0].with_extension("failed");
|
||||
std::fs::rename(&disk_paths[0], &failed_disk_path).expect("failed to detach disk_paths[0]");
|
||||
std::fs::remove_dir_all(&disk_paths[0]).expect("failed to delete all contents under disk_paths[0]");
|
||||
std::fs::create_dir_all(&disk_paths[0]).expect("failed to recreate disk_paths[0] directory");
|
||||
assert!(!format_path.exists(), "replacement disk already contains format.json before heal");
|
||||
println!("✅ Deleted format.json on disk: {:?}", disk_paths[0]);
|
||||
|
||||
let (_format_result, format_error) = heal_storage.heal_format(false).await.expect("failed to run heal_format");
|
||||
|
||||
@@ -67,7 +67,6 @@ const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
|
||||
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
||||
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
|
||||
|
||||
@@ -160,7 +159,6 @@ pub struct InternodeMetricsSnapshot {
|
||||
pub operation_write_shutdown_errors_total: u64,
|
||||
pub signature_v1_fallback_total: u64,
|
||||
pub body_digest_fallback_total: u64,
|
||||
pub replay_scope_fallback_total: u64,
|
||||
pub replay_cache_overflow_total: u64,
|
||||
}
|
||||
|
||||
@@ -182,7 +180,6 @@ pub struct InternodeMetrics {
|
||||
msgpack_json_decode_error_total: AtomicU64,
|
||||
signature_v1_fallback_total: AtomicU64,
|
||||
body_digest_fallback_total: AtomicU64,
|
||||
replay_scope_fallback_total: AtomicU64,
|
||||
replay_cache_overflow_total: AtomicU64,
|
||||
}
|
||||
|
||||
@@ -434,13 +431,6 @@ impl InternodeMetrics {
|
||||
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is
|
||||
/// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`.
|
||||
pub fn record_replay_scope_fallback(&self) {
|
||||
self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
|
||||
/// full. Overflow fails closed, so a sustained non-zero rate means
|
||||
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
|
||||
@@ -498,7 +488,6 @@ impl InternodeMetrics {
|
||||
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
|
||||
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
|
||||
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
|
||||
replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
|
||||
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
@@ -521,7 +510,6 @@ impl InternodeMetrics {
|
||||
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
|
||||
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
@@ -899,19 +887,6 @@ mod tests {
|
||||
assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_fallback_counter_updates_snapshot_and_resets() {
|
||||
let metrics = InternodeMetrics::default();
|
||||
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
|
||||
|
||||
metrics.record_replay_scope_fallback();
|
||||
metrics.record_replay_scope_fallback();
|
||||
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 2);
|
||||
|
||||
metrics.reset_for_test();
|
||||
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_peer_flips_offline_after_threshold_and_back_online() {
|
||||
// Unique addr keeps this independent of the process-global registry / other tests.
|
||||
|
||||
@@ -25,9 +25,6 @@ keywords = ["rustfs", "openstack", "keystone", "authentication", "s3"]
|
||||
categories = ["authentication", "web-programming"]
|
||||
authors.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true, features = ["rt", "sync"] }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
|
||||
@@ -44,9 +44,7 @@ argon2 = { workspace = true }
|
||||
chacha20poly1305 = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
base64 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
subtle = { workspace = true }
|
||||
zeroize = { workspace = true, features = ["derive"] }
|
||||
|
||||
# Configuration and storage
|
||||
@@ -57,7 +55,7 @@ tempfile = { workspace = true }
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
|
||||
# Additional dependencies
|
||||
md-5 = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
arc-swap = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
|
||||
+126
-209
@@ -35,6 +35,7 @@ use std::collections::HashMap;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// Reject key identifiers that would not name a single file directly inside the key
|
||||
@@ -76,6 +77,8 @@ const LOCAL_KMS_ARGON2_P_COST: u32 = 1;
|
||||
/// Local KMS client that stores keys in local files
|
||||
pub struct LocalKmsClient {
|
||||
config: LocalConfig,
|
||||
/// In-memory cache of loaded keys for performance
|
||||
key_cache: RwLock<HashMap<String, MasterKeyInfo>>,
|
||||
/// Master encryption key for encrypting stored keys
|
||||
master_cipher: Option<Aes256Gcm>,
|
||||
/// Legacy pre-beta.9 master cipher for reading pre-Argon2 key files
|
||||
@@ -136,14 +139,13 @@ impl LocalKmsClient {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
let client = Self {
|
||||
Ok(Self {
|
||||
config,
|
||||
key_cache: RwLock::new(HashMap::new()),
|
||||
master_cipher,
|
||||
legacy_master_cipher,
|
||||
dek_crypto: AesDekCrypto::new(),
|
||||
};
|
||||
client.validate_existing_keys().await?;
|
||||
Ok(client)
|
||||
})
|
||||
}
|
||||
|
||||
/// Derive a 256-bit key from the master key string using a persistent Argon2id salt.
|
||||
@@ -191,35 +193,10 @@ impl LocalKmsClient {
|
||||
|
||||
let mut salt = [0u8; LOCAL_KMS_MASTER_KEY_SALT_LEN];
|
||||
rand::rng().fill(&mut salt[..]);
|
||||
let temp_path = config
|
||||
.key_dir
|
||||
.join(format!("{LOCAL_KMS_MASTER_KEY_SALT_FILE}.tmp-{}", uuid::Uuid::new_v4()));
|
||||
fs::write(&temp_path, salt).await?;
|
||||
Self::set_file_permissions(&temp_path, config.file_permissions).await?;
|
||||
match fs::hard_link(&temp_path, &salt_path).await {
|
||||
Ok(()) => {
|
||||
if let Err(error) = fs::remove_file(&temp_path).await {
|
||||
warn!(path = ?temp_path, %error, "Failed to remove Local KMS salt temporary file");
|
||||
}
|
||||
debug!(path = ?salt_path, "Local KMS master key salt created");
|
||||
Ok(salt)
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
let bytes = fs::read(&salt_path).await?;
|
||||
bytes.try_into().map_err(|_| {
|
||||
KmsError::configuration_error(format!(
|
||||
"Local KMS master key salt at {} must be exactly {} bytes",
|
||||
salt_path.display(),
|
||||
LOCAL_KMS_MASTER_KEY_SALT_LEN
|
||||
))
|
||||
})
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
Err(error.into())
|
||||
}
|
||||
}
|
||||
fs::write(&salt_path, salt).await?;
|
||||
Self::set_file_permissions(&salt_path, config.file_permissions).await?;
|
||||
debug!(path = ?salt_path, "Local KMS master key salt created");
|
||||
Ok(salt)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -265,12 +242,6 @@ impl LocalKmsClient {
|
||||
|
||||
let content = fs::read(&key_path).await?;
|
||||
let stored_key: StoredMasterKey = serde_json::from_slice(&content)?;
|
||||
if stored_key.key_id != key_id {
|
||||
return Err(KmsError::invalid_key(format!(
|
||||
"Local KMS key file identity mismatch: expected {key_id:?}, found {:?}",
|
||||
stored_key.key_id
|
||||
)));
|
||||
}
|
||||
|
||||
let encrypted_bytes = BASE64
|
||||
.decode(&stored_key.encrypted_key_material)
|
||||
@@ -355,43 +326,8 @@ impl LocalKmsClient {
|
||||
/// Save a master key to disk
|
||||
async fn save_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
|
||||
let key_path = self.master_key_path(&master_key.key_id)?;
|
||||
let content = self.encode_master_key(master_key, key_material)?;
|
||||
let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
|
||||
fs::write(&temp_path, &content).await?;
|
||||
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
|
||||
fs::rename(&temp_path, &key_path).await?;
|
||||
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn save_new_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
|
||||
let key_path = self.master_key_path(&master_key.key_id)?;
|
||||
let content = self.encode_master_key(master_key, key_material)?;
|
||||
let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
|
||||
fs::write(&temp_path, &content).await?;
|
||||
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
|
||||
|
||||
match fs::hard_link(&temp_path, &key_path).await {
|
||||
Ok(()) => {
|
||||
if let Err(error) = fs::remove_file(&temp_path).await {
|
||||
warn!(path = ?temp_path, %error, "Failed to remove Local KMS key temporary file");
|
||||
}
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key created");
|
||||
Ok(())
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
Err(KmsError::key_already_exists(&master_key.key_id))
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
Err(error.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<Vec<u8>> {
|
||||
// Encrypt key material if master cipher is available
|
||||
let (encrypted_key_material, nonce, at_rest_protection) = if let Some(ref cipher) = self.master_cipher {
|
||||
let mut nonce_bytes = [0u8; 12];
|
||||
rand::rng().fill(&mut nonce_bytes[..]);
|
||||
@@ -426,7 +362,17 @@ impl LocalKmsClient {
|
||||
at_rest_protection,
|
||||
};
|
||||
|
||||
serde_json::to_vec_pretty(&stored_key).map_err(Into::into)
|
||||
let content = serde_json::to_vec_pretty(&stored_key)?;
|
||||
|
||||
// Write to temporary file first, then rename for atomicity
|
||||
let temp_path = key_path.with_extension("tmp");
|
||||
fs::write(&temp_path, &content).await?;
|
||||
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
|
||||
|
||||
fs::rename(&temp_path, &key_path).await?;
|
||||
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the actual key material for a master key
|
||||
@@ -435,23 +381,6 @@ impl LocalKmsClient {
|
||||
Ok(key_material)
|
||||
}
|
||||
|
||||
async fn validate_existing_keys(&self) -> Result<()> {
|
||||
let mut entries = fs::read_dir(&self.config.key_dir).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
if path.extension().is_none_or(|extension| extension != "key") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let key_id = path
|
||||
.file_stem()
|
||||
.and_then(|stem| stem.to_str())
|
||||
.ok_or_else(|| KmsError::configuration_error("Local KMS key file name must be valid UTF-8"))?;
|
||||
self.decode_stored_key(key_id).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encrypt data using a master key
|
||||
async fn encrypt_with_master_key(&self, key_id: &str, plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
|
||||
// Load the actual master key material
|
||||
@@ -584,7 +513,11 @@ impl KmsClient for LocalKmsClient {
|
||||
let master_key = MasterKeyInfo::new_with_description(key_id.to_string(), algorithm.to_string(), Some(created_by), None);
|
||||
|
||||
// Save to disk
|
||||
self.save_new_master_key(&master_key, &key_material).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Cache the key
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
debug!(key_id, "Local KMS master key created");
|
||||
Ok(master_key)
|
||||
@@ -593,7 +526,23 @@ impl KmsClient for LocalKmsClient {
|
||||
async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> {
|
||||
debug!("Describing key: {}", key_id);
|
||||
|
||||
// Check cache first
|
||||
{
|
||||
let cache = self.key_cache.read().await;
|
||||
if let Some(master_key) = cache.get(key_id) {
|
||||
return Ok(master_key.clone().into());
|
||||
}
|
||||
}
|
||||
|
||||
// Load from disk
|
||||
let master_key = self.load_master_key(key_id).await?;
|
||||
|
||||
// Update cache
|
||||
{
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
}
|
||||
|
||||
Ok(master_key.into())
|
||||
}
|
||||
|
||||
@@ -653,6 +602,10 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key enabled");
|
||||
Ok(())
|
||||
}
|
||||
@@ -668,6 +621,10 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key disabled");
|
||||
Ok(())
|
||||
}
|
||||
@@ -689,6 +646,10 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key deletion scheduled");
|
||||
Ok(())
|
||||
}
|
||||
@@ -704,17 +665,31 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key deletion canceled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
|
||||
if !fs::try_exists(self.master_key_path(key_id)?).await? {
|
||||
return Err(KmsError::key_not_found(key_id));
|
||||
}
|
||||
Err(KmsError::invalid_operation(
|
||||
"Local KMS key rotation is unavailable until historical key versions can be retained",
|
||||
))
|
||||
debug!("Rotating key: {}", key_id);
|
||||
|
||||
let mut master_key = self.load_master_key(key_id).await?;
|
||||
master_key.version += 1;
|
||||
master_key.rotated_at = Some(Zoned::now());
|
||||
|
||||
// Generate new key material
|
||||
let key_material = generate_key_material(&master_key.algorithm)?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
debug!(key_id, "Local KMS key rotated");
|
||||
Ok(master_key)
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<()> {
|
||||
@@ -770,6 +745,11 @@ impl KmsBackend for LocalKmsBackend {
|
||||
async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
|
||||
let key_id = request.key_name.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
|
||||
|
||||
// `save_master_key` writes through a temp file and renames over the destination, so
|
||||
// creating a key under an existing name would replace its material and silently
|
||||
// destroy the ability to decrypt everything wrapped under it. The sibling
|
||||
// `KmsClient::create_key` has always refused this; the backend path did not, and
|
||||
// this is the path the admin API uses.
|
||||
if self.client.master_key_path(&key_id)?.exists() {
|
||||
return Err(KmsError::key_already_exists(&key_id));
|
||||
}
|
||||
@@ -787,8 +767,11 @@ impl KmsBackend for LocalKmsBackend {
|
||||
request.description.clone(),
|
||||
);
|
||||
|
||||
// Save to disk
|
||||
self.client.save_new_master_key(&master_key, &key_material).await?;
|
||||
// Save to disk and cache
|
||||
self.client.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.insert(key_id.clone(), master_key.clone());
|
||||
|
||||
master_key
|
||||
};
|
||||
@@ -905,6 +888,10 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.await
|
||||
.map_err(|e| KmsError::internal_error(format!("Failed to delete key file: {e}")))?;
|
||||
|
||||
// Remove from cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.remove(key_id);
|
||||
|
||||
debug!(key_id, "Local KMS key deleted immediately");
|
||||
|
||||
// Return success response for immediate deletion
|
||||
@@ -948,6 +935,10 @@ impl KmsBackend for LocalKmsBackend {
|
||||
|
||||
self.client.save_master_key(&master_key, &existing_key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
// Convert master_key to KeyMetadata for response
|
||||
let key_metadata = KeyMetadata {
|
||||
key_id: master_key.key_id.clone(),
|
||||
@@ -995,6 +986,10 @@ impl KmsBackend for LocalKmsBackend {
|
||||
|
||||
self.client.save_master_key(&master_key, &existing_key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
// Convert master_key to KeyMetadata for response
|
||||
let key_metadata = KeyMetadata {
|
||||
key_id: master_key.key_id.clone(),
|
||||
@@ -1194,18 +1189,6 @@ mod tests {
|
||||
.expect("stored encrypted key should deserialize");
|
||||
assert_eq!(stored.at_rest_protection, StoredKeyProtection::EncryptedMasterKey);
|
||||
assert_eq!(stored.nonce.len(), 12);
|
||||
|
||||
let wrong_master_error = match LocalKmsClient::new(LocalConfig {
|
||||
key_dir: client.config.key_dir.clone(),
|
||||
master_key: Some("wrong-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("wrong master key must fail initialization"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(wrong_master_error, KmsError::CryptographicError { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1245,57 +1228,17 @@ mod tests {
|
||||
master_key: None,
|
||||
file_permissions: Some(0o600),
|
||||
};
|
||||
let err = match LocalKmsClient::new(config).await {
|
||||
Ok(_) => panic!("initialization must reject an unreadable encrypted key"),
|
||||
Err(error) => error,
|
||||
};
|
||||
let client_without_master = LocalKmsClient::new(config)
|
||||
.await
|
||||
.expect("client without master key should still initialize in dev-mode tests");
|
||||
|
||||
let err = client_without_master
|
||||
.describe_key("encrypted-key", None)
|
||||
.await
|
||||
.expect_err("encrypted key should require a master key to read");
|
||||
assert!(err.to_string().contains("requires a configured master key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_key_rotation_is_rejected_without_overwriting_key_material() {
|
||||
let (client, _temp_dir) = create_test_client().await;
|
||||
let key_id = "rotation-key";
|
||||
client.create_key(key_id, "AES_256", None).await.expect("create key");
|
||||
let original_material = client.get_key_material(key_id).await.expect("load original material");
|
||||
|
||||
let error = client
|
||||
.rotate_key(key_id, None)
|
||||
.await
|
||||
.expect_err("rotation must remain unavailable without historical key versions");
|
||||
|
||||
assert!(matches!(error, KmsError::InvalidOperation { .. }));
|
||||
assert_eq!(
|
||||
client.get_key_material(key_id).await.expect("reload original material"),
|
||||
original_material
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn startup_rejects_key_file_with_mismatched_embedded_id() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("file-name", "AES_256", None).await.expect("create key");
|
||||
let key_path = client.master_key_path("file-name").expect("valid key id");
|
||||
let mut stored: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(&key_path).await.expect("read key file")).expect("decode key file");
|
||||
stored["key_id"] = serde_json::json!("embedded-name");
|
||||
fs::write(&key_path, serde_json::to_vec_pretty(&stored).expect("encode mismatched key"))
|
||||
.await
|
||||
.expect("write mismatched key");
|
||||
|
||||
let error = match LocalKmsClient::new(LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: None,
|
||||
file_permissions: Some(0o600),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("mismatched key identity must fail initialization"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(error, KmsError::InvalidKey { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_accepts_legacy_rfc3339_timestamp() {
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
@@ -1386,6 +1329,17 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("write beta.5 fixture");
|
||||
let mut explicit_protection = stored_key.clone();
|
||||
let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
|
||||
explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
|
||||
explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
|
||||
fs::write(
|
||||
temp_dir.path().join("beta5-explicit-key.key"),
|
||||
serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
|
||||
)
|
||||
.await
|
||||
.expect("write explicit-protection fixture");
|
||||
|
||||
let client = LocalKmsClient::new(LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: Some("beta5-test-master-key".to_string()),
|
||||
@@ -1399,34 +1353,24 @@ mod tests {
|
||||
.await
|
||||
.expect("decrypt beta.5 SHA-256 protected key");
|
||||
assert_eq!(material, vec![0x42; 32]);
|
||||
|
||||
let mut explicit_protection = stored_key.clone();
|
||||
let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
|
||||
explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
|
||||
explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
|
||||
fs::write(
|
||||
temp_dir.path().join("beta5-explicit-key.key"),
|
||||
serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
|
||||
)
|
||||
.await
|
||||
.expect("write explicit-protection fixture");
|
||||
let explicit_error = client
|
||||
.get_key_material("beta5-explicit-key")
|
||||
.await
|
||||
.expect_err("explicit current protection must not fall back to the beta.5 KDF");
|
||||
assert!(matches!(explicit_error, KmsError::CryptographicError { .. }));
|
||||
|
||||
let wrong_key_error = match LocalKmsClient::new(LocalConfig {
|
||||
let wrong_key_client = LocalKmsClient::new(LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: Some("wrong-beta5-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("wrong beta.5 master key must fail initialization"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(wrong_key_error, KmsError::CryptographicError { .. }));
|
||||
.expect("initialize local KMS with wrong beta.5 master key");
|
||||
let error = wrong_key_client
|
||||
.get_key_material("beta5-key")
|
||||
.await
|
||||
.expect_err("wrong beta.5 master key must not decrypt the fixture");
|
||||
assert!(matches!(error, KmsError::CryptographicError { .. }));
|
||||
}
|
||||
|
||||
/// R03-CAN-072 / R03-CAN-073: key identifiers arrive from request input, so every path
|
||||
@@ -1485,7 +1429,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// R07-CAN-103: creating a duplicate key must preserve its original material.
|
||||
/// R07-CAN-103: `save_master_key` writes a temp file and renames over the destination,
|
||||
/// so creating a key under an existing name would replace its material and silently
|
||||
/// destroy the ability to decrypt anything wrapped under it.
|
||||
#[tokio::test]
|
||||
async fn backend_create_key_refuses_to_replace_existing_key_material() {
|
||||
let temp_dir = TempDir::new().expect("Failed to create temp dir");
|
||||
@@ -1523,33 +1469,4 @@ mod tests {
|
||||
.expect("original key material must survive the refused create");
|
||||
assert_eq!(original, after, "existing key material must not be replaced");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_backend_create_allows_only_one_writer() {
|
||||
let temp_dir = TempDir::new().expect("create key directory");
|
||||
let config = || LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: Some("test-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
};
|
||||
let first = LocalKmsBackend {
|
||||
client: LocalKmsClient::new(config()).await.expect("create first client"),
|
||||
};
|
||||
let second = LocalKmsBackend {
|
||||
client: LocalKmsClient::new(config()).await.expect("create second client"),
|
||||
};
|
||||
let request = || CreateKeyRequest {
|
||||
key_name: Some("concurrent-key".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (first_result, second_result) = tokio::join!(first.create_key(request()), second.create_key(request()));
|
||||
assert_ne!(first_result.is_ok(), second_result.is_ok(), "exactly one create must succeed");
|
||||
let error = first_result
|
||||
.err()
|
||||
.or_else(|| second_result.err())
|
||||
.expect("one create must fail");
|
||||
assert!(matches!(error, KmsError::KeyAlreadyExists { .. }));
|
||||
assert_eq!(first.client.get_key_material("concurrent-key").await.expect("load key").len(), 32);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,8 +177,8 @@ mod tests {
|
||||
let service1 = manager.get_encryption_service().await.expect("Service should be available");
|
||||
|
||||
// Reconfigure to new service (zero-downtime)
|
||||
let mut config2 = config1;
|
||||
config2.timeout = std::time::Duration::from_secs(45);
|
||||
let temp_dir2 = TempDir::new().expect("Failed to create temp dir");
|
||||
let config2 = KmsConfig::local(temp_dir2.path().to_path_buf()).with_insecure_development_defaults();
|
||||
manager.reconfigure(config2).await.expect("Reconfiguration should succeed");
|
||||
|
||||
// Verify version 2
|
||||
|
||||
@@ -20,7 +20,6 @@ use crate::manager::KmsManager;
|
||||
use crate::types::*;
|
||||
use base64::Engine;
|
||||
use jiff::Zoned;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::random;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
@@ -28,12 +27,6 @@ use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tracing::debug;
|
||||
use zeroize::Zeroize;
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Data key for object encryption
|
||||
/// SECURITY: This struct automatically zeros sensitive key material when dropped
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -493,7 +486,8 @@ impl ObjectEncryptionService {
|
||||
|
||||
// Validate key MD5 if provided
|
||||
if let Some(expected_md5) = customer_key_md5 {
|
||||
let actual_md5_hex = md5_hex(customer_key);
|
||||
let actual_md5 = md5::compute(customer_key);
|
||||
let actual_md5_hex = format!("{actual_md5:x}");
|
||||
if actual_md5_hex != expected_md5.to_lowercase() {
|
||||
return Err(KmsError::validation_error("Customer key MD5 mismatch"));
|
||||
}
|
||||
|
||||
@@ -20,12 +20,10 @@ use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::service::ObjectEncryptionService;
|
||||
use arc_swap::ArcSwap;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use subtle::ConstantTimeEq;
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
@@ -33,53 +31,6 @@ const LOG_COMPONENT_KMS: &str = "kms";
|
||||
const LOG_SUBSYSTEM_SERVICE: &str = "service";
|
||||
const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
|
||||
|
||||
fn local_master_key_fingerprint(master_key: Option<&str>) -> [u8; 32] {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update([u8::from(master_key.is_some())]);
|
||||
if let Some(master_key) = master_key {
|
||||
digest.update(master_key.as_bytes());
|
||||
}
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
fn validate_local_transition(current: Option<&KmsConfig>, new: &KmsConfig) -> Result<()> {
|
||||
let Some(current) = current else {
|
||||
return Ok(());
|
||||
};
|
||||
let BackendConfig::Local(current_local) = ¤t.backend_config else {
|
||||
return Ok(());
|
||||
};
|
||||
let BackendConfig::Local(new_local) = &new.backend_config else {
|
||||
return Err(KmsError::configuration_error("Local KMS backend cannot be changed after configuration"));
|
||||
};
|
||||
|
||||
if current_local.key_dir != new_local.key_dir {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS key directory cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
if current_local.file_permissions != new_local.file_permissions {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS file permissions cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
if current.allow_insecure_dev_defaults != new.allow_insecure_dev_defaults {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS development mode cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
|
||||
let current_master_key = local_master_key_fingerprint(current_local.master_key.as_deref());
|
||||
let new_master_key = local_master_key_fingerprint(new_local.master_key.as_deref());
|
||||
if !bool::from(current_master_key.ct_eq(&new_master_key)) {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS master key cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// KMS service status
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub enum KmsServiceStatus {
|
||||
@@ -155,12 +106,7 @@ impl KmsServiceManager {
|
||||
|
||||
/// Configure KMS with new configuration
|
||||
pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
new_config.validate()?;
|
||||
{
|
||||
let config = self.config.read().await;
|
||||
validate_local_transition(config.as_ref(), &new_config)?;
|
||||
}
|
||||
|
||||
// Update configuration
|
||||
{
|
||||
@@ -308,9 +254,11 @@ impl KmsServiceManager {
|
||||
"KMS service reconfiguring"
|
||||
);
|
||||
new_config.validate()?;
|
||||
|
||||
// Configure with new config
|
||||
{
|
||||
let config = self.config.read().await;
|
||||
validate_local_transition(config.as_ref(), &new_config)?;
|
||||
let mut config = self.config.write().await;
|
||||
*config = Some(new_config.clone());
|
||||
}
|
||||
|
||||
// Create new service version without stopping old one
|
||||
@@ -320,11 +268,6 @@ impl KmsServiceManager {
|
||||
// Get old version for logging (lock-free read)
|
||||
let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version);
|
||||
|
||||
{
|
||||
let mut config = self.config.write().await;
|
||||
*config = Some(new_config);
|
||||
}
|
||||
|
||||
// Atomically switch to new service version (lock-free, instant CAS operation)
|
||||
// This is a true atomic operation - no waiting for locks, instant switch
|
||||
// Old service will be dropped when no more Arc references exist
|
||||
@@ -361,6 +304,8 @@ impl KmsServiceManager {
|
||||
Err(e) => {
|
||||
let err_msg = format!("Failed to reconfigure KMS: {e}");
|
||||
error!("{}", err_msg);
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Error(err_msg.clone());
|
||||
Err(KmsError::backend_error(&err_msg))
|
||||
}
|
||||
}
|
||||
@@ -532,105 +477,4 @@ mod tests {
|
||||
};
|
||||
assert!(static_config.secret_key.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forbidden_local_master_key_change_preserves_running_config_and_service() {
|
||||
use crate::types::{CreateKeyRequest, KeyUsage};
|
||||
use std::collections::HashMap;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let config = |master_key: &str| {
|
||||
let mut config = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
let BackendConfig::Local(local) = &mut config.backend_config else {
|
||||
panic!("local constructor must create local backend config");
|
||||
};
|
||||
local.master_key = Some(master_key.to_string());
|
||||
config.allow_insecure_dev_defaults = true;
|
||||
config
|
||||
};
|
||||
let manager = KmsServiceManager::new();
|
||||
manager
|
||||
.configure(config("working-master-key"))
|
||||
.await
|
||||
.expect("configure local KMS");
|
||||
manager.start().await.expect("start local KMS");
|
||||
manager
|
||||
.get_manager()
|
||||
.await
|
||||
.expect("running KMS manager")
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some("existing-key".to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
policy: None,
|
||||
tags: HashMap::new(),
|
||||
origin: None,
|
||||
})
|
||||
.await
|
||||
.expect("create encrypted key");
|
||||
let service_version = manager.get_service_version().await;
|
||||
|
||||
let error = manager
|
||||
.reconfigure(config("wrong-master-key"))
|
||||
.await
|
||||
.expect_err("local master key change must be rejected");
|
||||
|
||||
assert!(error.to_string().contains("master key cannot be changed"));
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
assert_eq!(manager.get_service_version().await, service_version);
|
||||
let current = manager.get_config().await.expect("working config must remain");
|
||||
let BackendConfig::Local(local) = current.backend_config else {
|
||||
panic!("working config must remain local");
|
||||
};
|
||||
assert_eq!(local.master_key.as_deref(), Some("working-master-key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configure_cannot_replace_existing_local_backend() {
|
||||
use base64::Engine as _;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let mut local = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
local.allow_insecure_dev_defaults = true;
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(local.clone()).await.expect("configure local KMS");
|
||||
|
||||
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
|
||||
let error = manager
|
||||
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
|
||||
.await
|
||||
.expect_err("existing local backend must be immutable");
|
||||
|
||||
assert!(error.to_string().contains("backend cannot be changed"));
|
||||
let current = manager.get_config().await.expect("local config must remain");
|
||||
assert!(matches!(current.backend_config, BackendConfig::Local(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconfigure_allows_safe_local_runtime_settings_only() {
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let mut initial = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
initial.allow_insecure_dev_defaults = true;
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(initial.clone()).await.expect("configure local KMS");
|
||||
manager.start().await.expect("start local KMS");
|
||||
|
||||
let mut updated = initial;
|
||||
updated.default_key_id = Some("evaluation-key".to_string());
|
||||
updated.timeout = std::time::Duration::from_secs(45);
|
||||
updated.enable_cache = false;
|
||||
manager
|
||||
.reconfigure(updated.clone())
|
||||
.await
|
||||
.expect("update safe local settings");
|
||||
|
||||
let current = manager.get_config().await.expect("updated config");
|
||||
assert_eq!(current.default_key_id, updated.default_key_id);
|
||||
assert_eq!(current.timeout, updated.timeout);
|
||||
assert!(!current.enable_cache);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -545,8 +545,6 @@ pub struct ScannerMetrics {
|
||||
pub collected_at: DateTime<Utc>,
|
||||
#[serde(rename = "current_cycle")]
|
||||
pub current_cycle: u64,
|
||||
#[serde(rename = "current_cycle_active", default, skip_serializing_if = "Option::is_none")]
|
||||
pub current_cycle_active: Option<bool>,
|
||||
#[serde(rename = "current_started")]
|
||||
pub current_started: DateTime<Utc>,
|
||||
#[serde(rename = "cycle_complete_times")]
|
||||
@@ -720,31 +718,7 @@ pub struct ScannerMetrics {
|
||||
}
|
||||
|
||||
impl ScannerMetrics {
|
||||
pub fn is_current_cycle_active(&self) -> bool {
|
||||
self.current_cycle_active.unwrap_or(self.current_cycle > 0)
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: &Self) {
|
||||
// Legacy nodes omit the activity field and use a non-zero cycle as
|
||||
// their active signal. New nodes publish explicit first-cycle and idle
|
||||
// states, including cycle zero.
|
||||
let self_cycle_active = self.is_current_cycle_active();
|
||||
let other_cycle_active = other.is_current_cycle_active();
|
||||
let self_cycle_authority = (
|
||||
self_cycle_active,
|
||||
self.current_cycle,
|
||||
self.cycles_completed_at.len(),
|
||||
self.cycles_completed_at.as_slice(),
|
||||
self.current_started,
|
||||
);
|
||||
let other_cycle_authority = (
|
||||
other_cycle_active,
|
||||
other.current_cycle,
|
||||
other.cycles_completed_at.len(),
|
||||
other.cycles_completed_at.as_slice(),
|
||||
other.current_started,
|
||||
);
|
||||
let other_cycle_is_authoritative = self_cycle_authority < other_cycle_authority;
|
||||
let other_is_newer = self.collected_at < other.collected_at;
|
||||
if other_is_newer {
|
||||
self.collected_at = other.collected_at;
|
||||
@@ -880,12 +854,15 @@ impl ScannerMetrics {
|
||||
self.ongoing_buckets = other.ongoing_buckets;
|
||||
}
|
||||
|
||||
if other_cycle_is_authoritative {
|
||||
if self.current_cycle < other.current_cycle {
|
||||
self.current_cycle = other.current_cycle;
|
||||
self.cycles_completed_at = other.cycles_completed_at.clone();
|
||||
self.current_started = other.current_started;
|
||||
}
|
||||
self.current_cycle_active = Some(self_cycle_active || other_cycle_active);
|
||||
|
||||
if other.cycles_completed_at.len() > self.cycles_completed_at.len() {
|
||||
self.cycles_completed_at = other.cycles_completed_at.clone();
|
||||
}
|
||||
|
||||
if !other.life_time_ops.is_empty() && self.life_time_ops.is_empty() {
|
||||
self.life_time_ops = other.life_time_ops.clone();
|
||||
@@ -954,13 +931,7 @@ impl Metrics {
|
||||
if let Some(scanner) = other.scanner.as_ref() {
|
||||
match self.scanner {
|
||||
Some(ref mut s_scanner) => s_scanner.merge(scanner),
|
||||
None => {
|
||||
let mut scanner = scanner.clone();
|
||||
if scanner.current_cycle_active.is_none() {
|
||||
scanner.current_cycle_active = Some(scanner.is_current_cycle_active());
|
||||
}
|
||||
self.scanner = Some(scanner);
|
||||
}
|
||||
None => self.scanner = Some(scanner.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1418,280 +1389,6 @@ pub struct Operations {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_serializes_cycle_active_presence() {
|
||||
let missing_value = serde_json::to_value(ScannerMetrics::default()).expect("scanner metrics should serialize");
|
||||
assert!(missing_value.get("current_cycle_active").is_none());
|
||||
let missing: ScannerMetrics =
|
||||
serde_json::from_value(missing_value).expect("older scanner metrics without cycle-active should decode");
|
||||
assert_eq!(missing.current_cycle_active, None);
|
||||
|
||||
let explicit_false_value = serde_json::to_value(ScannerMetrics {
|
||||
current_cycle_active: Some(false),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("scanner metrics with explicit cycle-active should serialize");
|
||||
assert_eq!(explicit_false_value["current_cycle_active"], serde_json::Value::Bool(false));
|
||||
let explicit_false: ScannerMetrics =
|
||||
serde_json::from_value(explicit_false_value).expect("explicit cycle-active should decode");
|
||||
assert_eq!(explicit_false.current_cycle_active, Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_prefers_an_active_first_cycle() {
|
||||
let collected_at = Utc::now();
|
||||
let idle_started = collected_at - chrono::Duration::hours(1);
|
||||
let active_started = collected_at - chrono::Duration::seconds(5);
|
||||
let mut scanner = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 0,
|
||||
current_cycle_active: Some(false),
|
||||
current_started: idle_started,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
scanner.merge(&ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle: 0,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: active_started,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_cycle, 0);
|
||||
assert_eq!(scanner.current_started, active_started);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metrics_merge_preserves_explicit_active_first_cycle() {
|
||||
let mut aggregated = Metrics::default();
|
||||
aggregated.merge(&Metrics {
|
||||
scanner: Some(ScannerMetrics {
|
||||
current_cycle_active: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let scanner = aggregated.scanner.expect("aggregated scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_cycle, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_preserves_legacy_nonzero_active_signal() {
|
||||
let collected_at = Utc::now();
|
||||
let mut scanner = ScannerMetrics {
|
||||
collected_at,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
scanner.merge(&ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle: 7,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_cycle, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metrics_merge_normalizes_first_legacy_scanner_snapshot() {
|
||||
let legacy = Metrics {
|
||||
scanner: Some(ScannerMetrics {
|
||||
current_cycle: 7,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let mut aggregated = Metrics::default();
|
||||
|
||||
aggregated.merge(&legacy);
|
||||
|
||||
let scanner = aggregated.scanner.expect("aggregated scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_cycle, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_preserves_explicit_inactive_nonzero_cycle() {
|
||||
let collected_at = Utc::now();
|
||||
let mut scanner = ScannerMetrics::default();
|
||||
|
||||
scanner.merge(&ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(false),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(false));
|
||||
assert_eq!(scanner.current_cycle, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_cycle_active_is_order_independent() {
|
||||
let collected_at = Utc::now();
|
||||
let legacy_active = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_started: collected_at - chrono::Duration::seconds(10),
|
||||
..Default::default()
|
||||
};
|
||||
let explicit_idle = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 0,
|
||||
current_cycle_active: Some(false),
|
||||
current_started: collected_at - chrono::Duration::hours(1),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut legacy_first = legacy_active.clone();
|
||||
legacy_first.merge(&explicit_idle);
|
||||
let mut legacy_second = explicit_idle.clone();
|
||||
legacy_second.merge(&legacy_active);
|
||||
assert_eq!(legacy_first.current_cycle_active, Some(true));
|
||||
assert_eq!(legacy_second.current_cycle_active, Some(true));
|
||||
assert_eq!(legacy_first.current_cycle, 7);
|
||||
assert_eq!(legacy_second.current_cycle, 7);
|
||||
|
||||
let explicit_inactive_nonzero = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(false),
|
||||
..Default::default()
|
||||
};
|
||||
let mut inactive_first = explicit_inactive_nonzero.clone();
|
||||
inactive_first.merge(&explicit_idle);
|
||||
let mut inactive_second = explicit_idle;
|
||||
inactive_second.merge(&explicit_inactive_nonzero);
|
||||
assert_eq!(inactive_first.current_cycle_active, Some(false));
|
||||
assert_eq!(inactive_second.current_cycle_active, Some(false));
|
||||
assert_eq!(inactive_first.current_cycle, 7);
|
||||
assert_eq!(inactive_second.current_cycle, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_cycle_authority_is_order_independent() {
|
||||
let collected_at = Utc::now();
|
||||
let completion = collected_at - chrono::Duration::minutes(1);
|
||||
let earlier_active = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: collected_at - chrono::Duration::seconds(10),
|
||||
cycles_completed_at: vec![completion],
|
||||
..Default::default()
|
||||
};
|
||||
let later_active = ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: collected_at - chrono::Duration::seconds(5),
|
||||
cycles_completed_at: vec![completion],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut earlier_first = earlier_active.clone();
|
||||
earlier_first.merge(&later_active);
|
||||
let mut later_first = later_active.clone();
|
||||
later_first.merge(&earlier_active);
|
||||
|
||||
assert_eq!(earlier_first.current_started, later_active.current_started);
|
||||
assert_eq!(later_first.current_started, later_active.current_started);
|
||||
assert_eq!(earlier_first.cycles_completed_at, later_active.cycles_completed_at);
|
||||
assert_eq!(later_first.cycles_completed_at, later_active.cycles_completed_at);
|
||||
|
||||
let stale_idle = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle_active: Some(false),
|
||||
current_started: collected_at - chrono::Duration::hours(1),
|
||||
..Default::default()
|
||||
};
|
||||
let completed_idle = ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle_active: Some(false),
|
||||
current_started: collected_at - chrono::Duration::seconds(5),
|
||||
cycles_completed_at: vec![completion],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut stale_first = stale_idle.clone();
|
||||
stale_first.merge(&completed_idle);
|
||||
let mut completed_first = completed_idle.clone();
|
||||
completed_first.merge(&stale_idle);
|
||||
|
||||
assert_eq!(stale_first.current_started, completed_idle.current_started);
|
||||
assert_eq!(completed_first.current_started, completed_idle.current_started);
|
||||
assert_eq!(stale_first.cycles_completed_at, completed_idle.cycles_completed_at);
|
||||
assert_eq!(completed_first.cycles_completed_at, completed_idle.cycles_completed_at);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_cycle_authority_is_associative() {
|
||||
let collected_at = Utc::now();
|
||||
let older_completion = collected_at - chrono::Duration::minutes(3);
|
||||
let last_completion = collected_at - chrono::Duration::minutes(1);
|
||||
let cycle_seven = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: collected_at - chrono::Duration::seconds(10),
|
||||
cycles_completed_at: vec![older_completion, last_completion],
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_eight = ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle: 8,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: collected_at - chrono::Duration::seconds(5),
|
||||
cycles_completed_at: vec![last_completion],
|
||||
..Default::default()
|
||||
};
|
||||
let newer_idle = ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::hours(1),
|
||||
current_cycle_active: Some(false),
|
||||
current_started: collected_at - chrono::Duration::hours(1),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut left_associative = cycle_seven.clone();
|
||||
left_associative.merge(&cycle_eight);
|
||||
left_associative.merge(&newer_idle);
|
||||
|
||||
let mut right_group = cycle_eight.clone();
|
||||
right_group.merge(&newer_idle);
|
||||
let mut right_associative = cycle_seven.clone();
|
||||
right_associative.merge(&right_group);
|
||||
|
||||
assert_eq!(left_associative.current_cycle, 8);
|
||||
assert_eq!(right_associative.current_cycle, 8);
|
||||
assert_eq!(left_associative.current_started, cycle_eight.current_started);
|
||||
assert_eq!(right_associative.current_started, cycle_eight.current_started);
|
||||
assert_eq!(left_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
|
||||
assert_eq!(right_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
|
||||
|
||||
for order in [
|
||||
[&cycle_seven, &cycle_eight, &newer_idle],
|
||||
[&cycle_seven, &newer_idle, &cycle_eight],
|
||||
[&cycle_eight, &cycle_seven, &newer_idle],
|
||||
[&cycle_eight, &newer_idle, &cycle_seven],
|
||||
[&newer_idle, &cycle_seven, &cycle_eight],
|
||||
[&newer_idle, &cycle_eight, &cycle_seven],
|
||||
] {
|
||||
let mut merged = ScannerMetrics::default();
|
||||
for scanner in order {
|
||||
merged.merge(scanner);
|
||||
}
|
||||
assert_eq!(merged.current_cycle_active, Some(true));
|
||||
assert_eq!(merged.current_cycle, 8);
|
||||
assert_eq!(merged.current_started, cycle_eight.current_started);
|
||||
assert_eq!(merged.cycles_completed_at, cycle_eight.cycles_completed_at);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_aggregates_partial_cycles_by_source() {
|
||||
let collected_at = Utc::now();
|
||||
|
||||
@@ -262,11 +262,11 @@ async fn obs_site_replication_stats() -> ReplicationStats {
|
||||
}
|
||||
|
||||
fn current_scanner_cycle_age_seconds(
|
||||
current_cycle_active: bool,
|
||||
current_cycle: u64,
|
||||
current_started: chrono::DateTime<Utc>,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> u64 {
|
||||
if !current_cycle_active {
|
||||
if current_cycle == 0 {
|
||||
0
|
||||
} else {
|
||||
now.signed_duration_since(current_started).num_seconds().max(0) as u64
|
||||
@@ -1090,7 +1090,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
let reference_time = metrics.cycles_completed_at.last().copied().unwrap_or(metrics.current_started);
|
||||
let last_activity_seconds = now.signed_duration_since(reference_time).num_seconds().max(0) as u64;
|
||||
let active_paths = metrics.active_scan_paths as u64;
|
||||
let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle_active, metrics.current_started, now);
|
||||
let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle, metrics.current_started, now);
|
||||
let current_scan_mode = scanner_scan_mode_code(&metrics.current_scan_mode);
|
||||
let current_cycle_age = current_cycle_age_seconds as f64;
|
||||
let last_cycle_duration = metrics.last_cycle_duration_seconds;
|
||||
@@ -1464,21 +1464,21 @@ mod tests {
|
||||
fn current_scanner_cycle_age_seconds_returns_zero_when_idle() {
|
||||
let now = Utc::now();
|
||||
|
||||
assert_eq!(current_scanner_cycle_age_seconds(false, now - chrono::Duration::seconds(30), now), 0);
|
||||
assert_eq!(current_scanner_cycle_age_seconds(0, now - chrono::Duration::seconds(30), now), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_scanner_cycle_age_seconds_clamps_future_start() {
|
||||
let now = Utc::now();
|
||||
|
||||
assert_eq!(current_scanner_cycle_age_seconds(true, now + chrono::Duration::seconds(30), now), 0);
|
||||
assert_eq!(current_scanner_cycle_age_seconds(4, now + chrono::Duration::seconds(30), now), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_scanner_cycle_age_seconds_reports_active_first_cycle_elapsed_time() {
|
||||
fn current_scanner_cycle_age_seconds_reports_active_elapsed_time() {
|
||||
let now = Utc::now();
|
||||
|
||||
assert_eq!(current_scanner_cycle_age_seconds(true, now - chrono::Duration::seconds(45), now), 45);
|
||||
assert_eq!(current_scanner_cycle_age_seconds(4, now - chrono::Duration::seconds(45), now), 45);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -25,9 +25,6 @@ description = "Protocol implementations for RustFS (FTPS, SFTP, etc.)"
|
||||
keywords = ["ftp", "sftp", "protocol", "storage", "rustfs"]
|
||||
categories = ["network-programming", "filesystem"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime", "dep:subtle"]
|
||||
@@ -47,7 +44,7 @@ swift = [
|
||||
"dep:tokio-util",
|
||||
"dep:serde",
|
||||
"dep:urlencoding",
|
||||
"dep:md-5",
|
||||
"dep:md5",
|
||||
"dep:quick-xml",
|
||||
"dep:hmac",
|
||||
"dep:sha1",
|
||||
@@ -111,7 +108,7 @@ http-body-util = { workspace = true, optional = true }
|
||||
tokio-util = { workspace = true, optional = true, features = ["rt", "io", "compat"] }
|
||||
serde = { workspace = true, optional = true, features = ["derive"] }
|
||||
urlencoding = { workspace = true, optional = true }
|
||||
md-5 = { workspace = true, optional = true }
|
||||
md5 = { workspace = true, optional = true }
|
||||
quick-xml = { workspace = true, optional = true, features = ["serialize"] }
|
||||
hmac = { workspace = true, optional = true }
|
||||
sha1 = { workspace = true, optional = true }
|
||||
|
||||
@@ -608,7 +608,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.put_object_calls.push(PutObjectCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
metadata: input.metadata,
|
||||
metadata: input.metadata.clone(),
|
||||
});
|
||||
let stall = inner.stall_put_object;
|
||||
let entered = inner.put_object_entered.clone();
|
||||
@@ -731,7 +731,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.create_multipart_calls.push(CreateMultipartCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
metadata: input.metadata,
|
||||
metadata: input.metadata.clone(),
|
||||
});
|
||||
}
|
||||
match self.inner.lock().expect("lock").create_multipart_upload.pop_front() {
|
||||
@@ -749,7 +749,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.upload_part_calls.push(UploadPartCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
upload_id: input.upload_id,
|
||||
upload_id: input.upload_id.to_string(),
|
||||
part_number: input.part_number,
|
||||
content_length: input.content_length,
|
||||
});
|
||||
@@ -787,7 +787,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.complete_multipart_calls.push(CompleteCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
upload_id: input.upload_id,
|
||||
upload_id: input.upload_id.to_string(),
|
||||
part_count,
|
||||
});
|
||||
}
|
||||
@@ -808,7 +808,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.abort_multipart_calls.push(AbortCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
upload_id: input.upload_id,
|
||||
upload_id: input.upload_id.to_string(),
|
||||
});
|
||||
}
|
||||
match self.inner.lock().expect("lock").abort_multipart_upload.pop_front() {
|
||||
|
||||
@@ -379,7 +379,9 @@ where
|
||||
.await
|
||||
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
|
||||
|
||||
let prefix_with_slash = prefix.clone().map(|p| if p.ends_with('/') { p } else { format!("{}/", p) });
|
||||
let prefix_with_slash = prefix
|
||||
.clone()
|
||||
.map(|p| if p.ends_with('/') { p.to_string() } else { format!("{}/", p) });
|
||||
|
||||
let list_input = ListObjectsV2Input::builder()
|
||||
.bucket(bucket)
|
||||
|
||||
@@ -371,7 +371,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
|
||||
|
||||
let ftps_user = FtpsUser {
|
||||
username: principal.username.clone(),
|
||||
name: identity.credentials.name,
|
||||
name: identity.credentials.name.clone(),
|
||||
session_context,
|
||||
};
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user