mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-01 02:52:15 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2600a50c5d | |||
| 10db415099 |
+14
-88
@@ -156,99 +156,16 @@ jobs:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Prepare test evidence
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
{
|
||||
echo "run_id=${GITHUB_RUN_ID}"
|
||||
echo "job=${GITHUB_JOB}"
|
||||
echo "runner=${RUNNER_NAME}"
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/test-and-lint/run-metadata.txt
|
||||
|
||||
# Clippy runs before the test pass: lint failures are the most common
|
||||
# CI-only breakage and should surface in minutes, not after 20+ minutes
|
||||
# of tests.
|
||||
- name: Run clippy lints
|
||||
run: cargo clippy --all-targets -- -D warnings
|
||||
|
||||
- name: Run nextest tests
|
||||
- name: Run tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
# Evidence sampler for issue #5394: the post-mortem pgrep below runs
|
||||
# only after `timeout` has already TERM'd the whole cargo process
|
||||
# group, so it cannot name a wedged process. Sample system and
|
||||
# process state every 60s instead; the last samples before the
|
||||
# timeout show what was stuck (rustc, linker, build script, memory
|
||||
# pressure, ...). The log rides along in the existing artifact.
|
||||
(
|
||||
while true; do
|
||||
{
|
||||
echo "=== $(date --utc --iso-8601=seconds)"
|
||||
echo "--- load"; cat /proc/loadavg
|
||||
echo "--- psi"; grep -H . /proc/pressure/* 2>/dev/null || true
|
||||
echo "--- mem"; free -m
|
||||
echo "--- disk"; df -h / /home/runner 2>/dev/null || df -h /
|
||||
echo "--- top-rss"
|
||||
ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss | head -15
|
||||
echo "--- build/test processes"
|
||||
ps -eo pid,ppid,stat,etime,rss,pcpu,args | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true
|
||||
echo "--- d-state (uninterruptible IO)"
|
||||
ps -eo pid,stat,etime,args | awk 'NR > 1 && $2 ~ /D/' || true
|
||||
echo
|
||||
} >> artifacts/test-and-lint/sampler.log 2>&1 || true
|
||||
sleep 60
|
||||
done
|
||||
) &
|
||||
sampler_pid=$!
|
||||
trap 'kill "${sampler_pid}" 2>/dev/null || true' EXIT
|
||||
set +e
|
||||
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
|
||||
cargo nextest run --profile ci --all --exclude e2e_test \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
|
||||
echo "exit_status=${status}"
|
||||
echo "finished_at=$(date --utc --iso-8601=seconds)"
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|nextest|target/.*/deps/' || true
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Run documentation tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
set +e
|
||||
timeout --verbose --signal=TERM --kill-after=30s 15m \
|
||||
cargo test --all --doc \
|
||||
2>&1 | tee artifacts/test-and-lint/doctest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo test --all --doc"
|
||||
echo "exit_status=${status}"
|
||||
echo "finished_at=$(date --utc --iso-8601=seconds)"
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
cargo nextest run --profile ci --all --exclude e2e_test
|
||||
cargo test --all --doc
|
||||
|
||||
# rustfs/backlog#1289: fail if a seed rule's log anchor no longer exists
|
||||
# verbatim in the source tree (log message drifted without updating the
|
||||
@@ -257,6 +174,15 @@ jobs:
|
||||
- name: Check log-analyzer rule anchors
|
||||
run: ./scripts/check_log_analyzer_rules.sh
|
||||
|
||||
- name: Upload test junit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}
|
||||
path: target/nextest/ci/junit.xml
|
||||
retention-days: 3
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Explicit gate for migration-critical suites. These tests already ran in
|
||||
# the full nextest pass above; a single filtered nextest invocation keeps
|
||||
# the named gate without rebuilding or re-running them one package at a time.
|
||||
@@ -402,7 +328,7 @@ jobs:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build debug binary
|
||||
run: cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
run: cargo build -p rustfs --bins
|
||||
|
||||
- name: Upload debug binary
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
@@ -432,7 +358,7 @@ jobs:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build debug binary with rio-v2
|
||||
run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
|
||||
run: cargo build -p rustfs --bins --features rio-v2
|
||||
|
||||
- name: Upload debug binary
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
|
||||
@@ -66,7 +66,7 @@ env:
|
||||
CARGO_TERM_COLOR: always
|
||||
REGISTRY_DOCKERHUB: rustfs/rustfs
|
||||
REGISTRY_GHCR: ghcr.io/${{ github.repository }}
|
||||
REGISTRY_QUAY: quay.io/rustfs/rustfs
|
||||
REGISTRY_QUAY: quay.io/${{ secrets.QUAY_USERNAME }}/rustfs
|
||||
DOCKER_PLATFORMS: linux/amd64,linux/arm64
|
||||
|
||||
jobs:
|
||||
|
||||
Generated
+8
-14
@@ -3671,13 +3671,12 @@ dependencies = [
|
||||
"flatbuffers",
|
||||
"flate2",
|
||||
"futures",
|
||||
"hex",
|
||||
"http 1.4.2",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"local-ip-address",
|
||||
"md-5 0.11.0",
|
||||
"md5",
|
||||
"opentelemetry-proto",
|
||||
"prost 0.14.4",
|
||||
"rand 0.10.2",
|
||||
@@ -5645,9 +5644,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lazy-regex"
|
||||
version = "3.6.1"
|
||||
version = "3.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4994ba703f78b083e2f7946dac9251abd83fd43a0365f030e99b69be5b4b9ef9"
|
||||
checksum = "6bae91019476d3ec7147de9aa291cadb6d870abf2f3015d2da73a90325ac1496"
|
||||
dependencies = [
|
||||
"lazy-regex-proc_macros",
|
||||
"once_cell",
|
||||
@@ -5656,9 +5655,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lazy-regex-proc_macros"
|
||||
version = "3.6.1"
|
||||
version = "3.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd97232314824e6dbef1918a871bb93f51070455e3715bf26e19a6d01aa977a0"
|
||||
checksum = "4de9c1e1439d8b7b3061b2d209809f447ca33241733d9a3c01eabf2dc8d94358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -8891,7 +8890,7 @@ dependencies = [
|
||||
"libmimalloc-sys",
|
||||
"libsystemd",
|
||||
"matchit 0.9.2",
|
||||
"md-5 0.11.0",
|
||||
"md5",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"mimalloc",
|
||||
@@ -9450,10 +9449,9 @@ dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.0",
|
||||
"chacha20poly1305",
|
||||
"hex",
|
||||
"insta",
|
||||
"jiff",
|
||||
"md-5 0.11.0",
|
||||
"md5",
|
||||
"moka",
|
||||
"rand 0.10.2",
|
||||
"reqwest",
|
||||
@@ -9462,7 +9460,6 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"subtle",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
@@ -9720,7 +9717,7 @@ dependencies = [
|
||||
"hyper-util",
|
||||
"ipnetwork",
|
||||
"libunftp",
|
||||
"md-5 0.11.0",
|
||||
"md5",
|
||||
"percent-encoding",
|
||||
"proptest",
|
||||
"quick-xml",
|
||||
@@ -9735,7 +9732,6 @@ dependencies = [
|
||||
"rustfs-policy",
|
||||
"rustfs-rio",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-tls-runtime",
|
||||
"rustfs-trusted-proxies",
|
||||
"rustfs-utils",
|
||||
@@ -10131,7 +10127,6 @@ dependencies = [
|
||||
"bytes",
|
||||
"convert_case 0.11.0",
|
||||
"crc-fast",
|
||||
"criterion",
|
||||
"flate2",
|
||||
"futures",
|
||||
"hex-simd",
|
||||
@@ -11833,7 +11828,6 @@ dependencies = [
|
||||
"futures-util",
|
||||
"libc",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
|
||||
@@ -265,6 +265,7 @@ memmap2 = "0.9.11"
|
||||
lz4 = "1.28.1"
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
md5 = "0.8.1"
|
||||
mime_guess = "2.0.5"
|
||||
moka = { version = "0.12.15" }
|
||||
netif = "0.1.6"
|
||||
|
||||
@@ -25,9 +25,6 @@ keywords = ["checksum-calculation", "verification", "integrity", "authenticity",
|
||||
categories = ["web-programming", "development-tools", "network-programming"]
|
||||
documentation = "https://docs.rs/rustfs-checksums/latest/rustfs_checksum/"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
crc-fast = { workspace = true }
|
||||
|
||||
+33
-145
@@ -1114,8 +1114,6 @@ pub struct ScannerLastMinute {
|
||||
pub struct ScannerMetricsReport {
|
||||
pub collected_at: DateTime<Utc>,
|
||||
pub current_cycle: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_active: bool,
|
||||
pub current_started: DateTime<Utc>,
|
||||
pub cycles_completed_at: Vec<DateTime<Utc>>,
|
||||
pub ongoing_buckets: usize,
|
||||
@@ -2053,7 +2051,7 @@ impl Metrics {
|
||||
pub fn record_scanner_transition_failed(&self, count: u64) {
|
||||
self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
|
||||
self.record_scanner_source_failed(ScannerWorkSource::Lifecycle, count);
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
self.record_last_cycle_scanner_source_work(ScannerWorkSource::Lifecycle, ScannerSourceWorkUpdate::failed(count));
|
||||
}
|
||||
}
|
||||
@@ -2338,21 +2336,6 @@ impl Metrics {
|
||||
*self.cycle_info.write().await = cycle;
|
||||
}
|
||||
|
||||
/// Publish a scanner cycle and its work-accounting baseline as one state transition.
|
||||
pub async fn start_scan_cycle_work_with_cycle(&self, cycle: CurrentCycle) -> ScanCycleWorkSnapshot {
|
||||
let mut current_cycle = self.cycle_info.write().await;
|
||||
let snapshot = self.start_scan_cycle_work();
|
||||
*current_cycle = Some(cycle);
|
||||
snapshot
|
||||
}
|
||||
|
||||
/// Publish the completed work snapshot and idle cycle state as one state transition.
|
||||
pub async fn finish_scan_cycle_work_with_cycle(&self, start: ScanCycleWorkSnapshot, cycle: CurrentCycle) {
|
||||
let mut current_cycle = self.cycle_info.write().await;
|
||||
self.finish_scan_cycle_work(start);
|
||||
*current_cycle = Some(cycle);
|
||||
}
|
||||
|
||||
/// Read the current cycle record.
|
||||
pub async fn get_cycle(&self) -> Option<CurrentCycle> {
|
||||
self.cycle_info.read().await.clone()
|
||||
@@ -2481,7 +2464,7 @@ impl Metrics {
|
||||
&self.current_scan_cycle_replication_repair_work_start,
|
||||
&replication_repair_snapshot,
|
||||
);
|
||||
self.current_scan_cycle_work_active.store(true, Ordering::Release);
|
||||
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
|
||||
snapshot
|
||||
}
|
||||
|
||||
@@ -2493,11 +2476,11 @@ impl Metrics {
|
||||
self.record_scan_cycle_work(work);
|
||||
self.record_scan_cycle_source_work(&source_work);
|
||||
self.record_scan_cycle_replication_repair_work(&replication_repair_work);
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Release);
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -2763,41 +2746,13 @@ impl Metrics {
|
||||
pub async fn report(&self) -> ScannerMetricsReport {
|
||||
let mut m = ScannerMetricsReport::default();
|
||||
|
||||
let has_cycle = {
|
||||
let cycle = self.cycle_info.read().await;
|
||||
let has_cycle = if let Some(cycle) = cycle.as_ref() {
|
||||
m.current_cycle = cycle.current;
|
||||
m.cycles_completed_at = cycle.cycle_completed.clone();
|
||||
m.current_started = cycle.started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
|
||||
if m.current_cycle_active {
|
||||
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
|
||||
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
let current_replication_repair_work =
|
||||
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
|
||||
m.current_cycle_objects_scanned = current_work.objects_scanned;
|
||||
m.current_cycle_directories_scanned = current_work.directories_scanned;
|
||||
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
|
||||
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
|
||||
m.current_cycle_yield_events = current_work.yield_events;
|
||||
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
|
||||
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_ilm_actions = current_work.ilm_actions;
|
||||
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
|
||||
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
|
||||
m.current_cycle_heal_objects = current_work.heal_objects;
|
||||
m.current_cycle_replication_checks = current_work.replication_checks;
|
||||
m.current_cycle_usage_saves = current_work.usage_saves;
|
||||
m.current_cycle_source_work = self.scanner_source_work_snapshots(¤t_source_work);
|
||||
m.current_cycle_replication_repair =
|
||||
self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work);
|
||||
}
|
||||
has_cycle
|
||||
let has_cycle = if let Some(cycle) = self.get_cycle().await {
|
||||
m.current_cycle = cycle.current;
|
||||
m.cycles_completed_at = cycle.cycle_completed;
|
||||
m.current_started = cycle.started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
|
||||
@@ -2838,6 +2793,28 @@ impl Metrics {
|
||||
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
|
||||
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
|
||||
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
|
||||
if self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
|
||||
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
let current_replication_repair_work =
|
||||
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
|
||||
m.current_cycle_objects_scanned = current_work.objects_scanned;
|
||||
m.current_cycle_directories_scanned = current_work.directories_scanned;
|
||||
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
|
||||
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
|
||||
m.current_cycle_yield_events = current_work.yield_events;
|
||||
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
|
||||
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_ilm_actions = current_work.ilm_actions;
|
||||
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
|
||||
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
|
||||
m.current_cycle_heal_objects = current_work.heal_objects;
|
||||
m.current_cycle_replication_checks = current_work.replication_checks;
|
||||
m.current_cycle_usage_saves = current_work.usage_saves;
|
||||
m.current_cycle_source_work = self.scanner_source_work_snapshots(¤t_source_work);
|
||||
m.current_cycle_replication_repair = self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work);
|
||||
}
|
||||
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
|
||||
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
|
||||
m.last_cycle_result_code = last_cycle_result as u64;
|
||||
@@ -4165,8 +4142,6 @@ mod tests {
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert!(report.current_cycle_active);
|
||||
assert_eq!(report.current_cycle, 0);
|
||||
assert_eq!(report.current_cycle_objects_scanned, 7);
|
||||
assert_eq!(report.current_cycle_directories_scanned, 3);
|
||||
assert_eq!(report.current_cycle_bucket_drive_scans, 2);
|
||||
@@ -4183,8 +4158,6 @@ mod tests {
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert!(!report.current_cycle_active);
|
||||
assert_eq!(report.current_cycle, 0);
|
||||
assert_eq!(report.current_cycle_objects_scanned, 0);
|
||||
assert_eq!(report.current_cycle_directories_scanned, 0);
|
||||
assert_eq!(report.current_cycle_bucket_drive_scans, 0);
|
||||
@@ -4211,91 +4184,6 @@ mod tests {
|
||||
assert_eq!(report.last_cycle_usage_saves, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scan_cycle_activity_and_cycle_state_publish_together() {
|
||||
let metrics = Metrics::new();
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let active_cycle = CurrentCycle {
|
||||
current: 12,
|
||||
next: 13,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let cycle_state = metrics.cycle_info.read().await;
|
||||
let mut start_transition = Box::pin(metrics.start_scan_cycle_work_with_cycle(active_cycle));
|
||||
let waker = std::task::Waker::noop();
|
||||
let mut context = std::task::Context::from_waker(waker);
|
||||
assert!(start_transition.as_mut().poll(&mut context).is_pending());
|
||||
assert!(!metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
|
||||
drop(cycle_state);
|
||||
|
||||
let start = start_transition.await;
|
||||
let active = metrics.report().await;
|
||||
assert!(active.current_cycle_active);
|
||||
assert_eq!(active.current_cycle, 12);
|
||||
assert_eq!(active.current_started, cycle_started);
|
||||
|
||||
let idle_cycle = CurrentCycle {
|
||||
current: 0,
|
||||
next: 13,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_state = metrics.cycle_info.read().await;
|
||||
let mut finish_transition = Box::pin(metrics.finish_scan_cycle_work_with_cycle(start, idle_cycle));
|
||||
assert!(finish_transition.as_mut().poll(&mut context).is_pending());
|
||||
assert!(metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
|
||||
drop(cycle_state);
|
||||
|
||||
finish_transition.await;
|
||||
let idle = metrics.report().await;
|
||||
assert!(!idle.current_cycle_active);
|
||||
assert_eq!(idle.current_cycle, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_keeps_cycle_identity_and_work_in_one_snapshot() {
|
||||
let metrics = Metrics::new();
|
||||
let cycle_ten = CurrentCycle {
|
||||
current: 10,
|
||||
next: 11,
|
||||
started: Utc::now() - chrono::Duration::seconds(10),
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
|
||||
|
||||
let paths = metrics.current_paths.write().await;
|
||||
let mut report = Box::pin(metrics.report());
|
||||
let waker = std::task::Waker::noop();
|
||||
let mut context = std::task::Context::from_waker(waker);
|
||||
assert!(report.as_mut().poll(&mut context).is_pending());
|
||||
|
||||
metrics
|
||||
.finish_scan_cycle_work_with_cycle(cycle_ten_start, CurrentCycle { current: 0, ..cycle_ten })
|
||||
.await;
|
||||
let cycle_eleven_start = metrics
|
||||
.start_scan_cycle_work_with_cycle(CurrentCycle {
|
||||
current: 11,
|
||||
next: 12,
|
||||
started: Utc::now(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
|
||||
|
||||
drop(paths);
|
||||
let snapshot = report.await;
|
||||
|
||||
assert_eq!(snapshot.current_cycle, 10);
|
||||
assert_eq!(snapshot.current_cycle_objects_scanned, 1);
|
||||
|
||||
metrics
|
||||
.finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_ilm_actions_ignore_global_ilm_work() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -10,9 +10,6 @@ description = "Shared concurrency contract types for RustFS - workload admission
|
||||
keywords = ["rustfs", "concurrency", "admission", "backpressure", "workers"]
|
||||
categories = ["concurrency", "filesystem"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# Internal crates
|
||||
rustfs-io-core = { workspace = true }
|
||||
|
||||
@@ -116,27 +116,6 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
|
||||
/// Default: bitrot verification is enabled on GetObject reads (do not skip).
|
||||
pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
|
||||
|
||||
/// Request writing the complete remote-tier version state into object metadata.
|
||||
///
|
||||
/// This remains ineffective until
|
||||
/// [`ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED`] is also enabled.
|
||||
pub const ENV_TIER_REMOTE_VERSION_STATE_WRITE: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE";
|
||||
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested fleet-wide confirmation for
|
||||
/// [`ENV_TIER_REMOTE_VERSION_STATE_WRITE`].
|
||||
///
|
||||
/// This flag is an operational contract, not automatic capability discovery.
|
||||
/// Operators may enable it only after every node that can write or read
|
||||
/// transitioned object metadata supports the remote version-state schema and
|
||||
/// semantics. Keeping the confirmation separate makes a single-node request or
|
||||
/// a writer whose local opt-in is removed fail closed.
|
||||
pub const ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||
|
||||
// =============================================================================
|
||||
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
||||
// =============================================================================
|
||||
@@ -638,15 +617,3 @@ pub const ENV_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: &str = "RUSTFS_OBJ
|
||||
|
||||
/// Default read-ahead disable concurrency threshold: 4.
|
||||
pub const DEFAULT_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
mod remote_version_state_tests {
|
||||
#[test]
|
||||
fn remote_version_state_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_TIER_REMOTE_VERSION_STATE_WRITE, "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -506,35 +506,8 @@ pub struct ReplicationStats {
|
||||
}
|
||||
|
||||
impl ReplicationStats {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
let Self {
|
||||
pending_size,
|
||||
replicated_size,
|
||||
failed_size,
|
||||
failed_count,
|
||||
pending_count,
|
||||
missed_threshold_size,
|
||||
after_threshold_size,
|
||||
missed_threshold_count,
|
||||
after_threshold_count,
|
||||
replicated_count,
|
||||
} = self;
|
||||
|
||||
*pending_size == 0
|
||||
&& *replicated_size == 0
|
||||
&& *failed_size == 0
|
||||
&& *failed_count == 0
|
||||
&& *pending_count == 0
|
||||
&& *missed_threshold_size == 0
|
||||
&& *after_threshold_size == 0
|
||||
&& *missed_threshold_count == 0
|
||||
&& *after_threshold_count == 0
|
||||
&& *replicated_count == 0
|
||||
}
|
||||
|
||||
#[deprecated(note = "use is_empty instead")]
|
||||
pub fn empty(&self) -> bool {
|
||||
self.is_empty()
|
||||
self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -547,19 +520,16 @@ pub struct ReplicationAllStats {
|
||||
}
|
||||
|
||||
impl ReplicationAllStats {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
let Self {
|
||||
replica_size,
|
||||
replica_count,
|
||||
targets,
|
||||
} = self;
|
||||
|
||||
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
|
||||
}
|
||||
|
||||
#[deprecated(note = "use is_empty instead")]
|
||||
pub fn empty(&self) -> bool {
|
||||
self.is_empty()
|
||||
if self.replica_size != 0 && self.replica_count != 0 {
|
||||
return false;
|
||||
}
|
||||
for v in self.targets.values() {
|
||||
if !v.empty() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -813,7 +783,7 @@ impl DataUsageCache {
|
||||
return Some(root);
|
||||
}
|
||||
let mut flat = self.flatten(&root);
|
||||
if flat.replication_stats.as_ref().is_some_and(ReplicationAllStats::is_empty) {
|
||||
if flat.replication_stats.as_ref().is_some_and(|stats| stats.empty()) {
|
||||
flat.replication_stats = None;
|
||||
}
|
||||
Some(flat)
|
||||
@@ -1612,126 +1582,6 @@ mod tests {
|
||||
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_stats_empty_checks_every_field() {
|
||||
type SetField = fn(&mut ReplicationStats);
|
||||
|
||||
let cases: [(&str, SetField); 10] = [
|
||||
("pending_size", |stats| stats.pending_size = 1),
|
||||
("replicated_size", |stats| stats.replicated_size = 1),
|
||||
("failed_size", |stats| stats.failed_size = 1),
|
||||
("failed_count", |stats| stats.failed_count = 1),
|
||||
("pending_count", |stats| stats.pending_count = 1),
|
||||
("missed_threshold_size", |stats| stats.missed_threshold_size = 1),
|
||||
("after_threshold_size", |stats| stats.after_threshold_size = 1),
|
||||
("missed_threshold_count", |stats| stats.missed_threshold_count = 1),
|
||||
("after_threshold_count", |stats| stats.after_threshold_count = 1),
|
||||
("replicated_count", |stats| stats.replicated_count = 1),
|
||||
];
|
||||
|
||||
assert!(ReplicationStats::default().is_empty());
|
||||
for (field, set_nonzero) in cases {
|
||||
let mut stats = ReplicationStats::default();
|
||||
set_nonzero(&mut stats);
|
||||
assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_all_stats_empty_checks_aggregate_fields_independently() {
|
||||
let cases = [
|
||||
(
|
||||
"replica_size",
|
||||
ReplicationAllStats {
|
||||
replica_size: 1,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"replica_count",
|
||||
ReplicationAllStats {
|
||||
replica_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
];
|
||||
|
||||
assert!(ReplicationAllStats::default().is_empty());
|
||||
for (field, stats) in cases {
|
||||
assert!(!stats.is_empty(), "{field} must make aggregate replication stats non-empty");
|
||||
}
|
||||
|
||||
let empty_targets = ReplicationAllStats {
|
||||
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
|
||||
|
||||
let stats = ReplicationAllStats {
|
||||
targets: HashMap::from([
|
||||
("arn:test:empty".to_string(), ReplicationStats::default()),
|
||||
(
|
||||
"arn:test:non-empty".to_string(),
|
||||
ReplicationStats {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!stats.is_empty(), "a non-empty target must make aggregate replication stats non-empty");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_recursive_prunes_empty_and_preserves_pending_replication_stats() {
|
||||
let root = hash_path("bucket");
|
||||
let child = hash_path("bucket/child");
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace_hashed(&root, &None, &DataUsageEntry::default());
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
replication_stats: Some(ReplicationAllStats::default()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.size_recursive("bucket")
|
||||
.expect("bucket usage should flatten")
|
||||
.replication_stats
|
||||
.is_none()
|
||||
);
|
||||
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:pending".to_string(),
|
||||
ReplicationStats {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let flattened = cache.size_recursive("bucket").expect("bucket usage should flatten");
|
||||
let replication = flattened
|
||||
.replication_stats
|
||||
.expect("pending-only replication stats must survive pruning");
|
||||
|
||||
assert_eq!(replication.targets["arn:test:pending"].pending_count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_usage_cache_merge_adds_missing_child() {
|
||||
let mut base = DataUsageCache::default();
|
||||
|
||||
@@ -70,8 +70,7 @@ walkdir.workspace = true
|
||||
base64 = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
hex = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
prost.workspace = true
|
||||
sha2 = { workspace = true }
|
||||
|
||||
@@ -24,10 +24,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||
use serial_test::serial;
|
||||
use sha2::Sha256;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::info;
|
||||
|
||||
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
||||
@@ -71,9 +70,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn content_md5_base64(body: &[u8]) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(body);
|
||||
let digest = hasher.finalize();
|
||||
let digest = md5::compute(body);
|
||||
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
|
||||
}
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ use hyper::body::Incoming;
|
||||
use hyper::server::conn::http1;
|
||||
use hyper::service::service_fn;
|
||||
use hyper_util::rt::{TokioIo, TokioTimer};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use s3s::access::{S3Access, S3AccessContext};
|
||||
use s3s::auth::SimpleAuth;
|
||||
use s3s::dto::{
|
||||
@@ -828,25 +827,13 @@ fn ensure_body_growth(current: usize, added: usize) -> S3Result {
|
||||
|
||||
async fn md5_digest(body: Bytes, permit: OwnedSemaphorePermit) -> S3Result<([u8; 16], OwnedSemaphorePermit)> {
|
||||
if body.len() < 1024 * 1024 {
|
||||
return Ok((md5_bytes(body), permit));
|
||||
return Ok((md5::compute(body).0, permit));
|
||||
}
|
||||
tokio::task::spawn_blocking(move || (md5_bytes(body), permit))
|
||||
tokio::task::spawn_blocking(move || (md5::compute(body).0, permit))
|
||||
.await
|
||||
.map_err(|error| s3s::s3_error!(InternalError, "MD5 worker failed: {error}"))
|
||||
}
|
||||
|
||||
fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result {
|
||||
let total_bytes = state
|
||||
.total_bytes
|
||||
@@ -1018,7 +1005,7 @@ impl S3 for FakeBackend {
|
||||
Some(value) => value,
|
||||
None => {
|
||||
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
|
||||
hex::encode(digest)
|
||||
format!("{:x}", md5::Digest(digest))
|
||||
}
|
||||
};
|
||||
let version = ObjectVersion {
|
||||
@@ -1221,7 +1208,7 @@ impl S3 for FakeBackend {
|
||||
}
|
||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
|
||||
let e_tag = hex::encode(digest);
|
||||
let e_tag = format!("{:x}", md5::Digest(digest));
|
||||
let mut state = lock(&self.store);
|
||||
let existing_bytes = state
|
||||
.uploads
|
||||
@@ -1349,7 +1336,7 @@ impl S3 for FakeBackend {
|
||||
.collect();
|
||||
let (body, digests, _body_permits) = assemble_multipart(assembly_parts, total_len, _body_permits).await?;
|
||||
let part_count = requested.len();
|
||||
let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{}-{part_count}", md5_hex(digests)));
|
||||
let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{:x}-{part_count}", md5::compute(digests)));
|
||||
let version = ObjectVersion {
|
||||
version_id: upload.version_id.clone(),
|
||||
body,
|
||||
|
||||
@@ -30,7 +30,6 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
@@ -69,9 +68,7 @@ pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
|
||||
}
|
||||
|
||||
pub fn sse_customer_key_md5_base64(key: &str) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(key.as_bytes());
|
||||
BASE64.encode(hasher.finalize())
|
||||
BASE64.encode(md5::compute(key).0)
|
||||
}
|
||||
|
||||
pub async fn kms_admin_request(
|
||||
|
||||
@@ -25,18 +25,12 @@ use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use md5::compute;
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{info, warn};
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Test encryption of zero-byte files (empty files)
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -300,7 +294,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
|
||||
info!("🔍 Testing invalid SSE-C key length");
|
||||
let invalid_short_key = "short"; // Too short
|
||||
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
|
||||
let invalid_key_md5 = md5_hex(invalid_short_key);
|
||||
let invalid_key_md5 = format!("{:x}", compute(invalid_short_key));
|
||||
|
||||
let invalid_key_result = s3_client
|
||||
.put_object()
|
||||
|
||||
@@ -26,7 +26,6 @@ use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
@@ -51,15 +50,7 @@ fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
}
|
||||
|
||||
fn sse_customer_key_md5_base64(key: &str) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(key.as_bytes());
|
||||
base64::engine::general_purpose::STANDARD.encode(hasher.finalize())
|
||||
}
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
base64::engine::general_purpose::STANDARD.encode(md5::compute(key).0)
|
||||
}
|
||||
|
||||
/// Env var consumed by the local SSE-S3 DEK provider when KMS is not configured.
|
||||
@@ -5673,7 +5664,7 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let archive = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await;
|
||||
let expected_etag = format!("\"{}\"", md5_hex(&archive));
|
||||
let expected_etag = format!("\"{:x}\"", md5::compute(&archive));
|
||||
|
||||
let response = client
|
||||
.put_object()
|
||||
|
||||
@@ -23,8 +23,7 @@ use rustfs_protos::{
|
||||
proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
|
||||
SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, node_service_server::NodeService,
|
||||
node_service_server::NodeService,
|
||||
},
|
||||
};
|
||||
use std::pin::Pin;
|
||||
@@ -105,27 +104,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseRenewRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseReleaseRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
|
||||
let request = request.into_inner();
|
||||
let args: LockRequest = match serde_json::from_str(&request.args) {
|
||||
|
||||
@@ -95,7 +95,6 @@ const MANUAL_ASYNC_PARALLEL_OBJECTS: usize = 64;
|
||||
const MANUAL_ACTIVE_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_RESTART_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT: StdDuration = StdDuration::from_secs(15);
|
||||
const MANUAL_TRANSITION_CANCEL_BARRIER_ENV: &str = "RUSTFS_E2E_MANUAL_TRANSITION_CANCEL_BARRIER";
|
||||
const MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT: StdDuration = StdDuration::from_secs(90);
|
||||
const MANUAL_RESTART_RECOVERY_TIMEOUT: StdDuration = StdDuration::from_secs(80);
|
||||
const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
|
||||
@@ -1685,15 +1684,8 @@ async fn test_manual_transition_async_active_cancel_reports_terminal_cancelled()
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
@@ -103,12 +102,10 @@ impl Intercept for ResponseHeaderCapture {
|
||||
|
||||
fn customer_key(byte: u8) -> CustomerKey {
|
||||
let raw = [byte; 32];
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(raw);
|
||||
CustomerKey {
|
||||
raw: String::from_utf8_lossy(&raw).into_owned(),
|
||||
encoded: base64::engine::general_purpose::STANDARD.encode(raw),
|
||||
md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()),
|
||||
md5: base64::engine::general_purpose::STANDARD.encode(md5::compute(raw).0),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -127,8 +127,8 @@ pub mod bucket {
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with,
|
||||
init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
update_bucket_targets_under_transaction_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -305,8 +305,7 @@ pub mod disk {
|
||||
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
|
||||
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
|
||||
@@ -554,7 +554,6 @@ async fn delete_free_version_remote_object(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let version_id_exact = validate_transition_remote_version(oi)?;
|
||||
let identity = tier_destination_id_from_metadata(&oi.user_defined)?
|
||||
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
@@ -563,7 +562,7 @@ async fn delete_free_version_remote_object(
|
||||
&oi.transitioned_object.tier,
|
||||
identity,
|
||||
tier_config_mgr,
|
||||
version_id_exact,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -4202,23 +4201,6 @@ pub async fn get_transitioned_object_reader(
|
||||
get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr).await
|
||||
}
|
||||
|
||||
fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::Error> {
|
||||
let version = oi.transitioned_object.version_id.as_str();
|
||||
match oi.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state is unknown",
|
||||
)),
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled if version.is_empty() => Ok(false),
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull if version == "null" => Ok(true),
|
||||
rustfs_filemeta::TransitionVersionState::Exact if !version.is_empty() && version != "null" => Ok(true),
|
||||
_ => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state conflicts with its version ID",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -4228,7 +4210,6 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
opts: &ObjectOptions,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<GetObjectReader, std::io::Error> {
|
||||
validate_transition_remote_version(oi)?;
|
||||
let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?;
|
||||
let lease = match expected_identity {
|
||||
Some(identity) => {
|
||||
@@ -5525,7 +5506,6 @@ mod tests {
|
||||
tier: tier.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -5589,7 +5569,6 @@ mod tests {
|
||||
tier,
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -5612,70 +5591,6 @@ mod tests {
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 1,
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: String::new(),
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier,
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("unknown remote version state must fail before backend IO"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let backend = register_mock_tier(&manager, "WARM").await;
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect_err("unknown remote version state must fail before backend IO");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_remote_delete_requires_persisted_destination_identity() {
|
||||
@@ -5725,7 +5640,6 @@ mod tests {
|
||||
oi.transitioned_object.tier = "WARM".to_string();
|
||||
oi.transitioned_object.name = "remote/object".to_string();
|
||||
oi.transitioned_object.version_id = "remote-version".to_string();
|
||||
oi.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
let local_delete_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
|
||||
let legacy_err = delete_free_version_remote_object_then(&oi, &manager, {
|
||||
@@ -5736,8 +5650,7 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect_err("legacy free-version without identity must be retained");
|
||||
assert_eq!(legacy_err.kind(), std::io::ErrorKind::Other);
|
||||
assert_eq!(old_backend.remove_count().await, 0);
|
||||
assert!(legacy_err.to_string().contains("no durable backend identity"));
|
||||
assert_eq!(local_delete_calls.load(Ordering::Relaxed), 0);
|
||||
|
||||
let mut invalid_metadata = HashMap::new();
|
||||
@@ -5868,7 +5781,6 @@ mod tests {
|
||||
oi.transitioned_object.tier = "WARM".to_string();
|
||||
oi.transitioned_object.name = "remote/object".to_string();
|
||||
oi.transitioned_object.version_id = "remote-version".to_string();
|
||||
oi.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
"bucket",
|
||||
@@ -5884,12 +5796,7 @@ mod tests {
|
||||
Ok(_) => panic!("identity-bound GET must reject a same-name tier rebind"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Other);
|
||||
let admin_err = err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<crate::client::admin_handler_utils::AdminError>())
|
||||
.expect("identity mismatch should retain the typed tier error");
|
||||
assert_eq!(admin_err.code, crate::services::tier::tier::ERR_TIER_INVALID_CONFIG.code);
|
||||
assert!(err.to_string().contains("identity no longer matches"));
|
||||
assert_eq!(new_backend.get_count().await, 0);
|
||||
|
||||
oi.user_defined = Arc::new(HashMap::new());
|
||||
@@ -5995,8 +5902,7 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([1; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
|
||||
let err = state
|
||||
@@ -6107,8 +6013,7 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([1; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
|
||||
state
|
||||
@@ -8974,7 +8879,7 @@ mod tests {
|
||||
.await
|
||||
.expect("first worker result should persist");
|
||||
assert_eq!(first.state, ManualTransitionJobState::Running);
|
||||
assert_eq!(first.report.transition_completed, 0);
|
||||
assert_eq!(first.report.transition_completed, 1);
|
||||
assert_eq!(first.report.transition_failed, 0);
|
||||
|
||||
let duplicate = record_manual_transition_worker_result(
|
||||
@@ -8987,11 +8892,11 @@ mod tests {
|
||||
.await
|
||||
.expect("duplicate worker result should be idempotent");
|
||||
assert_eq!(duplicate.state, ManualTransitionJobState::Running);
|
||||
assert_eq!(duplicate.report.transition_completed, 0);
|
||||
assert_eq!(duplicate.report.transition_completed, 1);
|
||||
assert_eq!(duplicate.report.transition_failed, 0);
|
||||
|
||||
let second_key = manual_transition_worker_result_task_key(&bucket, "logs/b", None);
|
||||
let pending_record = record_manual_transition_worker_result(
|
||||
let final_record = record_manual_transition_worker_result(
|
||||
ecstore.clone(),
|
||||
job_id,
|
||||
&second_key,
|
||||
@@ -9000,14 +8905,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("second distinct worker result should persist");
|
||||
assert_eq!(pending_record.state, ManualTransitionJobState::Running);
|
||||
assert_eq!(pending_record.report.transition_completed, 0);
|
||||
assert_eq!(pending_record.report.transition_failed, 0);
|
||||
|
||||
let final_record =
|
||||
reconcile_manual_transition_worker_results(ecstore.clone(), job_id, ManualTransitionQueueSnapshot::default())
|
||||
.await
|
||||
.expect("worker result journal should reconcile");
|
||||
assert_eq!(final_record.state, ManualTransitionJobState::Partial);
|
||||
assert_eq!(final_record.report.transition_completed, 1);
|
||||
assert_eq!(final_record.report.transition_failed, 1);
|
||||
@@ -9042,7 +8940,7 @@ mod tests {
|
||||
.expect("worker result job record should save");
|
||||
|
||||
let task_key = manual_transition_worker_result_task_key(&bucket, "logs/fail", None);
|
||||
let pending_record = record_manual_transition_worker_result_with_reason(
|
||||
let final_record = record_manual_transition_worker_result_with_reason(
|
||||
ecstore.clone(),
|
||||
job_id,
|
||||
&task_key,
|
||||
@@ -9052,12 +8950,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("worker result with failure reason should persist");
|
||||
assert!(pending_record.report.tier_failure_by_reason.is_empty());
|
||||
assert_eq!(pending_record.report.transition_failed, 0);
|
||||
|
||||
let final_record = reconcile_manual_transition_worker_results(ecstore, job_id, ManualTransitionQueueSnapshot::default())
|
||||
.await
|
||||
.expect("worker failure reason should reconcile");
|
||||
assert_eq!(
|
||||
final_record
|
||||
.report
|
||||
@@ -10188,87 +10081,6 @@ mod tests {
|
||||
(backend, identity_hex)
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
|
||||
let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available")
|
||||
.backend_identity();
|
||||
let je = Jentry {
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: false,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
};
|
||||
|
||||
let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
.await
|
||||
.expect_err("unknown journal state must fail before backend IO");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_replay_deletes_confirmed_exact_provider_token() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
backend
|
||||
.set_put_remote_version(Some("provider-version-token".to_string()))
|
||||
.await;
|
||||
lease
|
||||
.put(
|
||||
"remote/object",
|
||||
crate::client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")),
|
||||
9,
|
||||
)
|
||||
.await
|
||||
.expect("confirmed remote candidate should be seeded");
|
||||
backend.set_remove_failure(true);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
let je = Jentry {
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "provider-version-token".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
};
|
||||
|
||||
crate::set_disk::cleanup_rejected_transition_upload_durably(
|
||||
&lease,
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
true,
|
||||
Some(ecstore.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("failed immediate cleanup should remain durable in the journal");
|
||||
assert!(backend.contains(&je.obj_name).await);
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
.await
|
||||
.expect("identity-bound exact journal must retry confirmed candidate cleanup");
|
||||
|
||||
assert!(!backend.contains(&je.obj_name).await);
|
||||
assert_eq!(backend.exact_remove_count(), 2);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), "provider-version-token".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed_recoverable_free_version(
|
||||
disk_paths: &[PathBuf],
|
||||
bucket: &str,
|
||||
@@ -10288,7 +10100,6 @@ mod tests {
|
||||
identity,
|
||||
);
|
||||
}
|
||||
let transition_version_id = Uuid::new_v4();
|
||||
let mut metadata = FileMeta::new();
|
||||
metadata
|
||||
.add_version(FileInfo {
|
||||
@@ -10297,9 +10108,7 @@ mod tests {
|
||||
version_id: Some(object_version_id),
|
||||
transition_status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
transitioned_objname: format!("remote/{bucket}/{object}"),
|
||||
transition_version_id: Some(transition_version_id),
|
||||
transition_version: Some(transition_version_id.to_string()),
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
transition_version_id: Some(Uuid::new_v4()),
|
||||
transition_tier: "WARM".to_string(),
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
metadata: transitioned_metadata,
|
||||
@@ -11318,25 +11127,23 @@ mod tests {
|
||||
|
||||
// Distinct payloads with distinct sizes: a mixed-generation reassembly
|
||||
// would produce bytes matching none of them (or fail the read outright).
|
||||
let candidates: Vec<Vec<u8>> = (0..2)
|
||||
let candidates: Vec<Vec<u8>> = (0..3)
|
||||
.map(|g| {
|
||||
let len = 4096 + g * 512;
|
||||
vec![b'a' + g as u8; len]
|
||||
})
|
||||
.collect();
|
||||
|
||||
let commit_barrier = MultipartCommitBarrier::install_for_arrivals(
|
||||
&bucket,
|
||||
object,
|
||||
MultipartCommitPause::PutPartBeforeLockAcquire,
|
||||
candidates.len(),
|
||||
);
|
||||
let commit_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
let start = Arc::new(tokio::sync::Barrier::new(candidates.len() + 1));
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
for payload in candidates.iter().cloned() {
|
||||
let store = ecstore.clone();
|
||||
let bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
let start = Arc::clone(&start);
|
||||
tasks.spawn(async move {
|
||||
start.wait().await;
|
||||
let mut data = PutObjReader::from_vec(payload.clone());
|
||||
store
|
||||
.put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
|
||||
@@ -11344,10 +11151,11 @@ mod tests {
|
||||
.map(|info| (info, payload))
|
||||
});
|
||||
}
|
||||
start.wait().await;
|
||||
|
||||
// Both writers finish streaming before racing for the uploadId commit
|
||||
// lock. Two generations are sufficient to exercise the mixed-shard
|
||||
// hazard, while each waiter sits behind at most one cross-disk rename.
|
||||
// The first writer holds the uploadId commit lock while the other
|
||||
// resends reach the same critical section. Releasing it proves the
|
||||
// handoff without depending on saturated CI disk latency.
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release();
|
||||
|
||||
|
||||
@@ -1646,14 +1646,40 @@ pub async fn record_manual_transition_worker_result_with_reason(
|
||||
job_id: Uuid,
|
||||
task_key: &str,
|
||||
result: ManualTransitionWorkerResult,
|
||||
_queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let result_record = ManualTransitionWorkerResultRecord::new_with_reason(job_id, task_key, result, failure_reason);
|
||||
if !save_manual_transition_worker_result_if_absent(api.clone(), &result_record).await? {
|
||||
return load_manual_transition_job_record(api, job_id).await;
|
||||
}
|
||||
load_manual_transition_job_record(api, job_id).await
|
||||
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if record.is_terminal() {
|
||||
return Ok(record);
|
||||
}
|
||||
record.record_worker_result_with_reason(result, queue_snapshot, failure_reason);
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
if record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(
|
||||
api.clone(),
|
||||
&record.scope_key,
|
||||
record.job_id,
|
||||
record.lease_id,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
|
||||
pub async fn renew_manual_transition_job_lease(
|
||||
|
||||
@@ -20,10 +20,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
@@ -45,7 +42,6 @@ const TIER_DELETE_JOURNAL_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
|
||||
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
|
||||
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -59,35 +55,24 @@ struct PersistedTierDeleteJournalEntry {
|
||||
backend_identity: Option<[u8; 32]>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
version_id_exact: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
version_state: Option<rustfs_filemeta::TransitionVersionState>,
|
||||
}
|
||||
|
||||
impl PersistedTierDeleteJournalEntry {
|
||||
fn from_jentry(je: &Jentry) -> Result<Self> {
|
||||
validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?;
|
||||
let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
let version = if legacy_unknown {
|
||||
if je.backend_identity.is_some() {
|
||||
fn from_jentry(je: &Jentry) -> Self {
|
||||
Self {
|
||||
version: if je.version_id_exact {
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION
|
||||
} else if je.backend_identity.is_some() {
|
||||
TIER_DELETE_JOURNAL_VERSION
|
||||
} else {
|
||||
1
|
||||
}
|
||||
} else {
|
||||
if je.backend_identity.is_none() {
|
||||
return Err(Error::other("new tier delete journal entry is missing its backend identity"));
|
||||
}
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION
|
||||
};
|
||||
Ok(Self {
|
||||
version,
|
||||
},
|
||||
obj_name: je.obj_name.clone(),
|
||||
version_id: je.version_id.clone(),
|
||||
tier_name: je.tier_name.clone(),
|
||||
backend_identity: je.backend_identity,
|
||||
version_id_exact: je.version_id_exact.then_some(true),
|
||||
version_state: (!legacy_unknown).then_some(je.version_state),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn into_jentry(self) -> Result<Jentry> {
|
||||
@@ -99,23 +84,19 @@ impl PersistedTierDeleteJournalEntry {
|
||||
if self.obj_name.is_empty() || self.tier_name.is_empty() {
|
||||
return Err(Error::other("tier delete journal entry is incomplete"));
|
||||
}
|
||||
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION
|
||||
&& self.version != TIER_DELETE_JOURNAL_STATE_VERSION
|
||||
&& self.version_id_exact.unwrap_or(false)
|
||||
{
|
||||
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION && self.version_id_exact.unwrap_or(false) {
|
||||
return Err(Error::other(
|
||||
"legacy tier delete journal entry has an unsupported exact version constraint",
|
||||
));
|
||||
}
|
||||
let (backend_identity, version_id_exact, version_state) = match self.version {
|
||||
1 => (None, false, rustfs_filemeta::TransitionVersionState::Unknown),
|
||||
let (backend_identity, version_id_exact) = match self.version {
|
||||
1 => (None, false),
|
||||
TIER_DELETE_JOURNAL_VERSION => (
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v2 entry is missing its backend identity"))?,
|
||||
),
|
||||
false,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
),
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION => {
|
||||
if self.version_id.is_empty() || self.version_id_exact != Some(true) {
|
||||
@@ -127,22 +108,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
.ok_or_else(|| Error::other("tier delete journal v3 entry is missing its backend identity"))?,
|
||||
),
|
||||
true,
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
)
|
||||
}
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION => {
|
||||
let state = self
|
||||
.version_state
|
||||
.ok_or_else(|| Error::other("tier delete journal v4 entry is missing its version state"))?;
|
||||
let exact = self.version_id_exact.unwrap_or(false);
|
||||
validate_version_state(state, &self.version_id, exact)?;
|
||||
(
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v4 entry is missing its backend identity"))?,
|
||||
),
|
||||
exact,
|
||||
state,
|
||||
)
|
||||
}
|
||||
version => return Err(Error::other(format!("unsupported tier delete journal version {version}"))),
|
||||
@@ -153,30 +118,10 @@ impl PersistedTierDeleteJournalEntry {
|
||||
tier_name: self.tier_name,
|
||||
backend_identity,
|
||||
version_id_exact,
|
||||
version_state,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_version_state(
|
||||
state: rustfs_filemeta::TransitionVersionState,
|
||||
version_id: &str,
|
||||
version_id_exact: bool,
|
||||
) -> Result<()> {
|
||||
use rustfs_filemeta::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull, Unknown};
|
||||
|
||||
let valid = match state {
|
||||
Unknown => !version_id_exact,
|
||||
KnownDisabled => version_id.is_empty() && !version_id_exact,
|
||||
SuspendedNull => version_id == "null" && version_id_exact,
|
||||
Exact => !version_id.is_empty() && version_id != "null" && version_id_exact,
|
||||
};
|
||||
if !valid {
|
||||
return Err(Error::other("tier delete journal version state conflicts with its version id"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TierDeleteJournalRecoveryStats {
|
||||
pub scanned: usize,
|
||||
@@ -214,7 +159,7 @@ pub(crate) fn decode_tier_delete_journal_entry(data: &[u8]) -> Result<Jentry> {
|
||||
}
|
||||
|
||||
pub(crate) fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
|
||||
serde_json::to_vec(&PersistedTierDeleteJournalEntry::from_jentry(je)?)
|
||||
serde_json::to_vec(&PersistedTierDeleteJournalEntry::from_jentry(je))
|
||||
.map_err(|err| Error::other(format!("encode tier delete journal failed: {err}")))
|
||||
}
|
||||
|
||||
@@ -264,35 +209,18 @@ where
|
||||
}
|
||||
|
||||
pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
if je.version_state == rustfs_filemeta::TransitionVersionState::Unknown {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"tier delete journal remote version state is unknown",
|
||||
));
|
||||
}
|
||||
let backend_identity = je
|
||||
.backend_identity
|
||||
.ok_or_else(|| std::io::Error::other("legacy tier delete journal has no durable backend identity"))?;
|
||||
if je.version_id_exact {
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
je.version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
@@ -478,9 +406,8 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, await_tier_delete_journal_recovery,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
tier_delete_journal_object_name,
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, await_tier_delete_journal_recovery, decode_tier_delete_journal_entry,
|
||||
encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity, tier_delete_journal_object_name,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::error::Result;
|
||||
@@ -493,8 +420,7 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([7; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -510,7 +436,6 @@ mod tests {
|
||||
assert_eq!(decoded.tier_name, je.tier_name);
|
||||
assert_eq!(decoded.backend_identity, je.backend_identity);
|
||||
assert_eq!(decoded.version_id_exact, je.version_id_exact);
|
||||
assert_eq!(decoded.version_state, je.version_state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -525,7 +450,7 @@ mod tests {
|
||||
let persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("exact journal JSON should decode");
|
||||
let decoded = decode_tier_delete_journal_entry(&encoded).expect("exact journal entry should decode");
|
||||
|
||||
assert_eq!(persisted["version"], TIER_DELETE_JOURNAL_STATE_VERSION);
|
||||
assert_eq!(persisted["version"], TIER_DELETE_JOURNAL_EXACT_VERSION);
|
||||
assert_eq!(persisted["version_id_exact"], true);
|
||||
assert!(decoded.version_id_exact);
|
||||
assert_ne!(tier_delete_journal_object_name(&exact), tier_delete_journal_object_name(&normalized));
|
||||
@@ -588,46 +513,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_rejects_conflicting_v4_version_states() {
|
||||
let identity = vec![7_u8; 32];
|
||||
let invalid = [
|
||||
("known-disabled", "unexpected", false),
|
||||
("suspended-null", "", true),
|
||||
("suspended-null", "null", false),
|
||||
("exact", "", true),
|
||||
("exact", "null", true),
|
||||
("exact", "version", false),
|
||||
("unknown", "version", true),
|
||||
];
|
||||
|
||||
for (state, version_id, exact) in invalid {
|
||||
let persisted = serde_json::json!({
|
||||
"version": TIER_DELETE_JOURNAL_STATE_VERSION,
|
||||
"obj_name": "remote/object",
|
||||
"version_id": version_id,
|
||||
"tier_name": "WARM",
|
||||
"backend_identity": identity,
|
||||
"version_id_exact": exact.then_some(true),
|
||||
"version_state": state,
|
||||
});
|
||||
let encoded = serde_json::to_vec(&persisted).expect("invalid journal fixture should encode");
|
||||
decode_tier_delete_journal_entry(&encoded).expect_err("conflicting v4 version state must fail closed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_journals_decode_with_unknown_version_state() {
|
||||
let v1 = br#"{"version":1,"obj_name":"remote/object","version_id":"opaque","tier_name":"WARM"}"#;
|
||||
let v2 = br#"{"version":2,"obj_name":"remote/object","version_id":"opaque","tier_name":"WARM","backend_identity":[7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]}"#;
|
||||
|
||||
for payload in [v1.as_slice(), v2.as_slice()] {
|
||||
let decoded = decode_tier_delete_journal_entry(payload).expect("legacy journal should decode");
|
||||
assert_eq!(decoded.version_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
assert!(!decoded.version_id_exact);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_path_is_stable_and_sanitized() {
|
||||
let je = journal_entry();
|
||||
@@ -645,8 +530,6 @@ mod tests {
|
||||
fn tier_delete_journal_paths_separate_legacy_and_backend_identities() {
|
||||
let mut legacy = journal_entry();
|
||||
legacy.backend_identity = None;
|
||||
legacy.version_id_exact = false;
|
||||
legacy.version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
let mut backend_a = journal_entry();
|
||||
backend_a.backend_identity = Some([1; 32]);
|
||||
let mut backend_b = journal_entry();
|
||||
@@ -692,8 +575,6 @@ mod tests {
|
||||
fn tier_delete_journal_without_transition_identity_stays_legacy() {
|
||||
let mut je = journal_entry();
|
||||
je.backend_identity = None;
|
||||
je.version_id_exact = false;
|
||||
je.version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
|
||||
let encoded = encode_tier_delete_journal_entry(&je).expect("legacy journal should remain encodable");
|
||||
let persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("journal JSON should decode");
|
||||
|
||||
@@ -185,7 +185,6 @@ struct ObjSweeper {
|
||||
transition_status: String,
|
||||
transition_tier: String,
|
||||
transition_version_id: String,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
remote_object: String,
|
||||
}
|
||||
|
||||
@@ -232,9 +231,7 @@ impl ObjSweeper {
|
||||
}
|
||||
|
||||
pub fn should_remove_remote_object(&self) -> Option<Jentry> {
|
||||
if self.transition_status != lifecycle::TRANSITION_COMPLETE
|
||||
|| self.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
if self.transition_status != lifecycle::TRANSITION_COMPLETE {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -252,11 +249,7 @@ impl ObjSweeper {
|
||||
version_id: self.transition_version_id.clone(),
|
||||
tier_name: self.transition_tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: matches!(
|
||||
self.transition_version_state,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: self.transition_version_state,
|
||||
version_id_exact: false,
|
||||
});
|
||||
}
|
||||
None
|
||||
@@ -293,7 +286,6 @@ pub struct Jentry {
|
||||
pub(crate) tier_name: String,
|
||||
pub(crate) backend_identity: Option<TierDestinationId>,
|
||||
pub(crate) version_id_exact: bool,
|
||||
pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
|
||||
}
|
||||
|
||||
impl ExpiryOp for Jentry {
|
||||
@@ -338,7 +330,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false).await
|
||||
}
|
||||
|
||||
async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
@@ -346,11 +338,8 @@ async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
validate_remote_version_id: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
if validate_remote_version_id {
|
||||
lease.validate_remote_version_id(rv_id)?;
|
||||
}
|
||||
lease.validate_remote_version_id(rv_id)?;
|
||||
|
||||
if remote_delete_breaker_is_open(Instant::now()).await {
|
||||
metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
|
||||
@@ -446,53 +435,7 @@ pub(crate) async fn delete_object_from_remote_tier_with_lease_idempotent(
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, version_id_exact, true).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idempotent(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
if rv_id.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"confirmed versioned transition candidate requires a non-empty remote version",
|
||||
));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if obj_name == "remote/empty-guard-probe" {
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, true, false).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
tier_name: &str,
|
||||
backend_identity: TierDestinationId,
|
||||
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, tier_name, backend_identity)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(obj_name, rv_id, &lease).await
|
||||
}
|
||||
|
||||
async fn delete_object_from_remote_tier_with_lease_idempotent_inner(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
validate_remote_version_id: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact, validate_remote_version_id)
|
||||
.await
|
||||
{
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact).await {
|
||||
Ok(()) => Ok(RemoteTierDeleteOutcome::Deleted),
|
||||
Err(err) if is_remote_tier_not_found_error(&err) => Ok(RemoteTierDeleteOutcome::AlreadyRemoved),
|
||||
Err(err) => {
|
||||
@@ -517,7 +460,6 @@ pub fn transitioned_delete_journal_entry(
|
||||
versioned: bool,
|
||||
suspended: bool,
|
||||
transitioned: &TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
let sweeper = ObjSweeper {
|
||||
version_id,
|
||||
@@ -526,7 +468,6 @@ pub fn transitioned_delete_journal_entry(
|
||||
transition_status: transitioned.status.clone(),
|
||||
transition_tier: transitioned.tier.clone(),
|
||||
transition_version_id: transitioned.version_id.clone(),
|
||||
transition_version_state,
|
||||
remote_object: transitioned.name.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -534,13 +475,8 @@ pub fn transitioned_delete_journal_entry(
|
||||
sweeper.should_remove_remote_object()
|
||||
}
|
||||
|
||||
pub fn transitioned_force_delete_journal_entry(
|
||||
transitioned: &TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
if transitioned.status != lifecycle::TRANSITION_COMPLETE
|
||||
|| transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject) -> Option<Jentry> {
|
||||
if transitioned.status != lifecycle::TRANSITION_COMPLETE {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -549,11 +485,7 @@ pub fn transitioned_force_delete_journal_entry(
|
||||
version_id: transitioned.version_id.clone(),
|
||||
tier_name: transitioned.tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: matches!(
|
||||
transition_version_state,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: transition_version_state,
|
||||
version_id_exact: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -562,14 +494,11 @@ mod test {
|
||||
use crate::client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, RemoteDeleteBreaker, RemoteTierDeleteOutcome,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure,
|
||||
};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -613,43 +542,6 @@ mod test {
|
||||
assert!(should_record_remote_delete_failure(&Error::other("NoSuchVersion")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transitioned_delete_journal_preserves_remote_version_state() {
|
||||
let cases = [
|
||||
(TransitionVersionState::Unknown, "legacy-version", None),
|
||||
(TransitionVersionState::KnownDisabled, "", Some(false)),
|
||||
(TransitionVersionState::SuspendedNull, "null", Some(true)),
|
||||
(TransitionVersionState::Exact, "opaque-version", Some(true)),
|
||||
];
|
||||
|
||||
for (state, version_id, expected_exact) in cases {
|
||||
let transitioned = TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: version_id.to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
status: lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let regular = transitioned_delete_journal_entry(None, false, false, &transitioned, state);
|
||||
let forced = transitioned_force_delete_journal_entry(&transitioned, state);
|
||||
|
||||
match expected_exact {
|
||||
Some(expected_exact) => {
|
||||
let regular = regular.expect("known version state should produce a regular delete journal entry");
|
||||
assert_eq!(regular.version_state, state);
|
||||
assert_eq!(regular.version_id_exact, expected_exact);
|
||||
let forced = forced.expect("known version state should produce a forced delete journal entry");
|
||||
assert_eq!(forced.version_state, state);
|
||||
assert_eq!(forced.version_id_exact, expected_exact);
|
||||
}
|
||||
None => {
|
||||
assert!(regular.is_none());
|
||||
assert!(forced.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn idempotent_remote_delete_treats_hooked_nosuchversion_as_already_removed() {
|
||||
@@ -772,55 +664,6 @@ mod test {
|
||||
assert_eq!(backend.remove_versions().await, vec![("remote/object".to_string(), String::new())]);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn confirmed_transition_cleanup_deletes_exact_provider_token() {
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
let manager = crate::services::tier::tier::TierConfigMgr::new();
|
||||
let backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await;
|
||||
let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM")
|
||||
.await
|
||||
.expect("test tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
drop(lease);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
|
||||
let outcome = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
"remote/object",
|
||||
"provider-version-token",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
.expect("confirmed upload compensation should delete the exact provider token");
|
||||
|
||||
assert_eq!(outcome, RemoteTierDeleteOutcome::Deleted);
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), "provider-version-token".to_string())]
|
||||
);
|
||||
|
||||
let err = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
"remote/empty-guard-probe",
|
||||
"",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
.expect_err("confirmed versioned cleanup must reject an empty token");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||
assert_eq!(backend.remove_count().await, 1);
|
||||
assert_eq!(
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.load(std::sync::atomic::Ordering::Relaxed),
|
||||
0,
|
||||
"empty remote versions must be rejected before exact cleanup dispatch"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breaker_opens_at_threshold_and_recovers_after_window() {
|
||||
let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
|
||||
|
||||
@@ -22,10 +22,7 @@ use uuid::Uuid;
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent_with_manager_and_identity;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -711,21 +708,6 @@ async fn recover_unknown_upload_outcome(
|
||||
TransitionCandidateProbe::UnversionedPresent => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
|
||||
{
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
&transaction.remote_object,
|
||||
&version_id,
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
&api.tier_config_mgr(),
|
||||
)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
|
||||
}
|
||||
|
||||
@@ -737,9 +737,6 @@ impl BucketMetadata {
|
||||
}
|
||||
BUCKET_TAGGING_CONFIG => {
|
||||
self.tagging_config_xml = data;
|
||||
// Drop the parsed form (like lifecycle above) so clearing the
|
||||
// payload can't leave stale parsed tags to be cached.
|
||||
self.tagging_config = None;
|
||||
self.tagging_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_QUOTA_CONFIG_FILE => {
|
||||
@@ -1321,30 +1318,6 @@ mod test {
|
||||
assert!(bm.lifecycle_config.is_none());
|
||||
}
|
||||
|
||||
/// Companion to the lifecycle case above. `parse_all_configs` skips empty
|
||||
/// XML rather than clearing, so without the explicit reset a cleared
|
||||
/// tagging config would keep serving the previously parsed tags.
|
||||
#[test]
|
||||
fn tagging_update_config_clears_parsed_config_on_delete() {
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
let tagging_xml = br#"<Tagging><TagSet><Tag><Key>env</Key><Value>prod</Value></Tag></TagSet></Tagging>"#;
|
||||
|
||||
bm.update_config(BUCKET_TAGGING_CONFIG, tagging_xml.to_vec())
|
||||
.expect("tagging config should update");
|
||||
bm.parse_all_configs().expect("tagging config should parse");
|
||||
assert!(bm.tagging_config.is_some());
|
||||
|
||||
bm.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
|
||||
.expect("tagging config delete should update metadata");
|
||||
|
||||
assert!(bm.tagging_config_xml.is_empty());
|
||||
assert!(bm.tagging_config.is_none());
|
||||
|
||||
// A re-parse must not resurrect them either.
|
||||
bm.parse_all_configs().expect("cleared tagging should parse");
|
||||
assert!(bm.tagging_config.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn marshal_msg_complete_example() {
|
||||
// Create a complete BucketMetadata with various configurations
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
|
||||
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use http::Uri;
|
||||
use rustfs_protos::{
|
||||
@@ -107,79 +107,10 @@ pub async fn node_service_time_out_client_no_auth(
|
||||
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
|
||||
}
|
||||
|
||||
/// The typed `tonic::Status` an internode RPC failure was converted from, if
|
||||
/// this error carries one.
|
||||
pub(crate) fn embedded_tonic_status(io_err: &std::io::Error) -> Option<&tonic::Status> {
|
||||
io_err.get_ref()?.downcast_ref::<RpcStatusError>().map(RpcStatusError::status)
|
||||
}
|
||||
|
||||
/// Decide whether a gRPC status reports a peer we cannot currently reach,
|
||||
/// rather than an application outcome from a live peer.
|
||||
///
|
||||
/// `Unavailable` is the one code that means "no service behind this channel":
|
||||
/// the client transport raises it when the connection is broken, and the
|
||||
/// server's own not-ready gates use it deliberately.
|
||||
///
|
||||
/// `Unknown` is the client transport's escape hatch for a cause it could not
|
||||
/// map to a code — tower's "Service was not ready: <cause>", an h2 error with
|
||||
/// no gRPC mapping. Our handlers never return it, so there its message is the
|
||||
/// only evidence available and the anchored needles decide.
|
||||
///
|
||||
/// Every other code is an answer from a live peer and is never a transport
|
||||
/// failure, whatever its message says. That distinction is the point of
|
||||
/// classifying by code: a peer relaying its own downstream trouble as
|
||||
/// `Internal("connection refused ...")`, or a handler interpolating a local
|
||||
/// `io::Error` into `Status::internal`, answered us perfectly well. Marking it
|
||||
/// offline over that text is the bug this classification replaces. Likewise a
|
||||
/// `Cancelled` "Timeout expired" from the per-RPC channel deadline means the
|
||||
/// peer is slow, not gone; gating it would turn load into a partition.
|
||||
pub(crate) fn is_network_like_status(status: &tonic::Status) -> bool {
|
||||
match status.code() {
|
||||
tonic::Code::Unavailable => true,
|
||||
tonic::Code::Unknown => message_has_network_needle(&status.to_string()),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Substring fallback for failures that only exist as text: dial errors
|
||||
/// wrapped by `get_client`, remote `error_info` payloads, and statuses
|
||||
/// flattened through `format!`. Needles must stay anchored to transport
|
||||
/// context — a bare word like "unavailable" also matches application text
|
||||
/// (e.g. a bucket named "unavailable-logs") and would take a healthy peer
|
||||
/// offline.
|
||||
pub(crate) fn message_has_network_needle(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
[
|
||||
"temporarily offline",
|
||||
"transport error",
|
||||
// tonic >= 0.14 renders Code::Unavailable as
|
||||
// `code: 'The service is currently unavailable'`.
|
||||
"code: 'the service is currently unavailable'",
|
||||
// RUSTFS_COMPAT_TODO(tonic-013-status-render): releases up to 1.0.0-alpha.38 shipped tonic 0.13, which rendered the same status as `status: Unavailable`, and peers relay that text in error_info. Remove after the minimum supported RustFS peer version ships tonic >= 0.14.
|
||||
"status: unavailable",
|
||||
"error trying to connect",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"broken pipe",
|
||||
"not connected",
|
||||
"unexpected eof",
|
||||
"timed out",
|
||||
"deadline has elapsed",
|
||||
"connection closed",
|
||||
"connection aborted",
|
||||
"tcp connect error",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| message.contains(needle))
|
||||
}
|
||||
|
||||
pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
||||
match err {
|
||||
DiskError::Timeout => true,
|
||||
DiskError::Io(io_err) => {
|
||||
if let Some(status) = embedded_tonic_status(io_err) {
|
||||
return is_network_like_status(status);
|
||||
}
|
||||
if matches!(
|
||||
io_err.kind(),
|
||||
ErrorKind::TimedOut
|
||||
@@ -193,7 +124,24 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
message_has_network_needle(&io_err.to_string())
|
||||
let message = io_err.to_string().to_ascii_lowercase();
|
||||
[
|
||||
"transport error",
|
||||
"unavailable",
|
||||
"error trying to connect",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"broken pipe",
|
||||
"not connected",
|
||||
"unexpected eof",
|
||||
"timed out",
|
||||
"deadline has elapsed",
|
||||
"connection closed",
|
||||
"connection aborted",
|
||||
"tcp connect error",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| message.contains(needle))
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
@@ -321,70 +269,6 @@ mod tests {
|
||||
let _ = provider.shutdown();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_like_disk_error_uses_typed_status_code() {
|
||||
// Transport-level Unavailable statuses justify retry/eviction.
|
||||
assert!(is_network_like_disk_error(&DiskError::from(tonic::Status::unavailable(
|
||||
"storage layer is not initialized"
|
||||
))));
|
||||
// Application statuses from a live peer must not look network-like,
|
||||
// even when their message contains transport-sounding words.
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::internal(
|
||||
"failed to heal bucket \"unavailable-logs\""
|
||||
))));
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::unauthenticated(
|
||||
"No valid auth token"
|
||||
))));
|
||||
// A slow peer that blew the per-RPC deadline is still answering.
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::cancelled("Timeout expired"))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_tonic_status_is_recovered_across_error_conversions() {
|
||||
// DiskError and StorageError share one wrapper, so a status keeps its
|
||||
// typed classification whichever error it was converted into first.
|
||||
let from_storage: DiskErrorType = crate::error::Error::from(tonic::Status::unavailable("peer gone")).into();
|
||||
let DiskError::Io(io_err) = &from_storage else {
|
||||
panic!("status-derived disk error should stay an Io error");
|
||||
};
|
||||
assert_eq!(embedded_tonic_status(io_err).map(|status| status.code()), Some(tonic::Code::Unavailable));
|
||||
|
||||
let from_disk = crate::error::Error::from(DiskError::from(tonic::Status::unavailable("peer gone")));
|
||||
let crate::error::Error::Io(io_err) = &from_disk else {
|
||||
panic!("status-derived storage error should stay an Io error");
|
||||
};
|
||||
assert_eq!(embedded_tonic_status(io_err).map(|status| status.code()), Some(tonic::Code::Unavailable));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_like_disk_error_ignores_transport_words_in_application_statuses() {
|
||||
// Same contract as the peer client: a status the peer answered with
|
||||
// is not a transport failure, so it must not drive a reconnect even
|
||||
// when its message describes one.
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::internal(
|
||||
"connection refused while dialing downstream backend"
|
||||
))));
|
||||
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::unauthenticated(
|
||||
"connection reset while validating token"
|
||||
))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_like_disk_error_requires_anchored_unavailable_needle() {
|
||||
// Regression: a bare "unavailable" needle used to match application
|
||||
// text such as a bucket name.
|
||||
assert!(!is_network_like_disk_error(&DiskError::other("bucket \"unavailable-logs\" not found")));
|
||||
// Anchored renderings of a flattened Unavailable status still match.
|
||||
assert!(is_network_like_disk_error(&DiskError::other(
|
||||
"code: 'The service is currently unavailable', message: \"peer gone\""
|
||||
)));
|
||||
assert!(is_network_like_disk_error(&DiskError::other(
|
||||
"status: Unavailable, message: \"peer gone\""
|
||||
)));
|
||||
assert!(is_network_like_disk_error(&DiskError::other("connection refused")));
|
||||
assert!(!is_network_like_disk_error(&DiskError::FileNotFound));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_keeps_auth_headers() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -443,16 +443,6 @@ pub fn set_tonic_canonical_body_digest<T>(request: &mut tonic::Request<T>, canon
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_tonic_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(
|
||||
request: &mut tonic::Request<T>,
|
||||
) -> std::io::Result<()> {
|
||||
let canonical_body = request
|
||||
.get_ref()
|
||||
.canonical_body()
|
||||
.map_err(|_| std::io::Error::other("RPC mutation body length cannot be represented"))?;
|
||||
set_tonic_canonical_body_digest(request, &canonical_body)
|
||||
}
|
||||
|
||||
pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
let version = request
|
||||
.metadata()
|
||||
@@ -476,7 +466,7 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify a mutating RPC's canonical body digest with a rolling-upgrade fallback.
|
||||
/// Verify a mutating disk RPC's canonical body digest with a rolling-upgrade fallback.
|
||||
///
|
||||
/// When the request carries a real (non-`UNSIGNED-PAYLOAD`) content SHA-256 it is verified exactly
|
||||
/// like [`verify_tonic_canonical_body_digest`]. The digest value is a member of the signed v2
|
||||
@@ -507,7 +497,7 @@ fn verify_tonic_mutation_body_digest_with_strictness<T>(
|
||||
Some(digest) if digest != UNSIGNED_PAYLOAD => verify_tonic_canonical_body_digest(request, canonical_body),
|
||||
_ => {
|
||||
// RUSTFS_COMPAT_TODO(disk-mutation-body-digest): accept digestless peers during rolling upgrades. Remove after the
|
||||
// minimum supported RustFS peer version body-binds every mutating RPC.
|
||||
// minimum supported RustFS peer version body-binds every mutating disk RPC.
|
||||
if strict {
|
||||
return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
|
||||
}
|
||||
@@ -687,28 +677,11 @@ mod tests {
|
||||
use crate::cluster::rpc::context_propagation::REQUEST_ID_HEADER;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use http::{HeaderMap, Method};
|
||||
use rustfs_protos::{
|
||||
CanonicalMutationBody as _, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
proto_gen::node_service::{Mss, SignalServiceRequest},
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::io::{self, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use time::OffsetDateTime;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
fn signal_service_request(signal: &str, sub_system: &str, dry_run: &str) -> SignalServiceRequest {
|
||||
SignalServiceRequest {
|
||||
vars: Some(Mss {
|
||||
value: HashMap::from([
|
||||
(PEER_RESTSIGNAL.to_string(), signal.to_string()),
|
||||
(PEER_RESTSUB_SYS.to_string(), sub_system.to_string()),
|
||||
(PEER_RESTDRY_RUN.to_string(), dry_run.to_string()),
|
||||
]),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
@@ -1623,72 +1596,6 @@ mod tests {
|
||||
assert_eq!(stripped.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_service_mutation_contract_rejects_tampering_and_replay() {
|
||||
ensure_test_rpc_secret();
|
||||
let body = signal_service_request("2", "scanner", "false")
|
||||
.canonical_body()
|
||||
.expect("small signal request should encode");
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("canonical body digest should be attached");
|
||||
let content_sha256 = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "SignalService", content_sha256)
|
||||
.expect("body-bound auth headers should build");
|
||||
request.metadata_mut().as_mut().extend(headers.clone());
|
||||
|
||||
assert!(
|
||||
verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers).is_ok(),
|
||||
"the first body-bound signal request must authenticate"
|
||||
);
|
||||
assert!(verify_tonic_mutation_body_digest(&request, &body).is_ok());
|
||||
|
||||
let tampered = signal_service_request("1", "scanner", "false")
|
||||
.canonical_body()
|
||||
.expect("small signal request should encode");
|
||||
let error = verify_tonic_mutation_body_digest(&request, &tampered)
|
||||
.expect_err("changing the signal must invalidate the signed digest");
|
||||
assert_eq!(error.to_string(), "RPC content SHA-256 mismatch");
|
||||
|
||||
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
|
||||
.expect_err("reusing the signal nonce must fail");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(rpc_body_digest_fallback_counter)]
|
||||
fn signal_service_mutation_contract_preserves_rollout_fallback_and_strictness() {
|
||||
let body = signal_service_request("2", "scanner", "false")
|
||||
.canonical_body()
|
||||
.expect("small signal request should encode");
|
||||
let before = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
let digestless = tonic::Request::new(());
|
||||
|
||||
assert!(
|
||||
verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, false).is_ok(),
|
||||
"old peers must remain compatible while the rollout gate is open"
|
||||
);
|
||||
assert_eq!(
|
||||
global_internode_metrics().snapshot().body_digest_fallback_total,
|
||||
before + 1,
|
||||
"accepted digestless signal requests must be visible in the fallback metric"
|
||||
);
|
||||
|
||||
let error = verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, true)
|
||||
.expect_err("strict mode must reject a digestless signal request");
|
||||
assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
|
||||
|
||||
let mut bound = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut bound, &body).expect("canonical body digest should be attached");
|
||||
bound
|
||||
.metadata_mut()
|
||||
.as_mut()
|
||||
.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
assert!(verify_tonic_mutation_body_digest_with_strictness(&bound, &body, true).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_rejects_replay_after_wall_clock_regression() {
|
||||
let now = Instant::now();
|
||||
|
||||
@@ -30,9 +30,9 @@ pub use client::{
|
||||
};
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
|
||||
verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -13,10 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, heal_control_time_out_client, node_service_time_out_client,
|
||||
tier_mutation_control_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
@@ -50,7 +50,6 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
|
||||
use rustfs_utils::XHost;
|
||||
use serde::{Deserialize, Serialize as _};
|
||||
@@ -70,6 +69,9 @@ use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const PEER_RESTSIGNAL: &str = "signal";
|
||||
pub const PEER_RESTSUB_SYS: &str = "sub-sys";
|
||||
pub const PEER_RESTDRY_RUN: &str = "dry-run";
|
||||
pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
|
||||
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
|
||||
const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
|
||||
@@ -469,21 +471,26 @@ impl PeerRestClient {
|
||||
self.offline.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Whether this failure means the peer is unreachable, so it should be
|
||||
/// gated offline and its connection evicted.
|
||||
///
|
||||
/// RPC failures are classified by their typed gRPC code first
|
||||
/// (`is_network_like_status`); an application error from a live peer must
|
||||
/// never take it offline no matter what its message says. The substring
|
||||
/// fallback only covers failures that exist purely as text, such as the
|
||||
/// dial errors `get_client` wraps.
|
||||
fn is_network_like_error(err: &Error) -> bool {
|
||||
if let Error::Io(io_err) = err
|
||||
&& let Some(status) = embedded_tonic_status(io_err)
|
||||
{
|
||||
return is_network_like_status(status);
|
||||
}
|
||||
message_has_network_needle(&err.to_string())
|
||||
let message = err.to_string().to_ascii_lowercase();
|
||||
[
|
||||
"temporarily offline",
|
||||
"transport error",
|
||||
"unavailable",
|
||||
"error trying to connect",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"broken pipe",
|
||||
"not connected",
|
||||
"unexpected eof",
|
||||
"timed out",
|
||||
"deadline has elapsed",
|
||||
"connection closed",
|
||||
"connection aborted",
|
||||
"tcp connect error",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| message.contains(needle))
|
||||
}
|
||||
|
||||
fn mark_offline_and_spawn_recovery(&self) {
|
||||
@@ -1158,11 +1165,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadBucketMetadataRequest {
|
||||
let request = Request::new(LoadBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
scanner_maintenance_change,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_bucket_metadata(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1182,10 +1188,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DeleteBucketMetadataRequest {
|
||||
let request = Request::new(DeleteBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_bucket_metadata(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1205,10 +1210,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DeletePolicyRequest {
|
||||
let request = Request::new(DeletePolicyRequest {
|
||||
policy_name: policy.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_policy(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1228,10 +1232,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadPolicyRequest {
|
||||
let request = Request::new(LoadPolicyRequest {
|
||||
policy_name: policy.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_policy(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1251,12 +1254,11 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadPolicyMappingRequest {
|
||||
let request = Request::new(LoadPolicyMappingRequest {
|
||||
user_or_group: user_or_group.to_string(),
|
||||
user_type,
|
||||
is_group,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_policy_mapping(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1276,10 +1278,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DeleteUserRequest {
|
||||
let request = Request::new(DeleteUserRequest {
|
||||
access_key: access_key.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_user(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1299,10 +1300,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DeleteServiceAccountRequest {
|
||||
let request = Request::new(DeleteServiceAccountRequest {
|
||||
access_key: access_key.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_service_account(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1322,11 +1322,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadUserRequest {
|
||||
let request = Request::new(LoadUserRequest {
|
||||
access_key: access_key.to_string(),
|
||||
temp,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_user(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1346,10 +1345,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadServiceAccountRequest {
|
||||
let request = Request::new(LoadServiceAccountRequest {
|
||||
access_key: access_key.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_service_account(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1369,10 +1367,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadGroupRequest {
|
||||
let request = Request::new(LoadGroupRequest {
|
||||
group: group.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_group(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1392,8 +1389,7 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(ReloadSiteReplicationConfigRequest {});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(ReloadSiteReplicationConfigRequest {});
|
||||
|
||||
let response = client.reload_site_replication_config(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1417,10 +1413,9 @@ impl PeerRestClient {
|
||||
vars.insert(PEER_RESTSIGNAL.to_string(), sig.to_string());
|
||||
vars.insert(PEER_RESTSUB_SYS.to_string(), sub_sys.to_string());
|
||||
vars.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string());
|
||||
let mut request = Request::new(SignalServiceRequest {
|
||||
let request = Request::new(SignalServiceRequest {
|
||||
vars: Some(Mss { value: vars }),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.signal_service(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1489,8 +1484,7 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(ReloadPoolMetaRequest {});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(ReloadPoolMetaRequest {});
|
||||
|
||||
let response = client.reload_pool_meta(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1511,10 +1505,9 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(StopRebalanceRequest {
|
||||
let request = Request::new(StopRebalanceRequest {
|
||||
expected_rebalance_id: expected_rebalance_id.unwrap_or_default().to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.stop_rebalance(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1535,8 +1528,7 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
|
||||
|
||||
let response = client.load_rebalance_meta(request).await?.into_inner();
|
||||
|
||||
@@ -1575,8 +1567,7 @@ impl PeerRestClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(StartDecommissionRequest { pool_indices });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(StartDecommissionRequest { pool_indices });
|
||||
|
||||
let response = client.start_decommission(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1599,8 +1590,7 @@ impl PeerRestClient {
|
||||
let pool_index = u32::try_from(pool_index)
|
||||
.map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(CancelDecommissionRequest { pool_index });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(CancelDecommissionRequest { pool_index });
|
||||
|
||||
let response = client.cancel_decommission(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1623,8 +1613,7 @@ impl PeerRestClient {
|
||||
let pool_index = u32::try_from(pool_index)
|
||||
.map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(ClearDecommissionRequest { pool_index });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let request = Request::new(ClearDecommissionRequest { pool_index });
|
||||
|
||||
let response = client.clear_decommission(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1644,14 +1633,10 @@ impl PeerRestClient {
|
||||
pub async fn load_transition_tier_config(&self) -> Result<()> {
|
||||
match self.load_transition_tier_config_outcome().await {
|
||||
TierConfigReloadOutcome::Success => Ok(()),
|
||||
// Only a reconnect-class failure says anything about the channel.
|
||||
// `finalize_result` marks the peer offline and evicts its connection
|
||||
// whenever the message looks network-like, and a peer that answered
|
||||
// and rejected the apply can easily report one ("release RPC failed:
|
||||
// transport error"). Routing those through here would gate a healthy,
|
||||
// responding peer out of every unrelated RPC.
|
||||
TierConfigReloadOutcome::TransientReconnect(err) => self.finalize_result(Err(err)).await,
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(err) | TierConfigReloadOutcome::Terminal(err) => Err(err),
|
||||
TierConfigReloadOutcome::TransientReconnect(err) | TierConfigReloadOutcome::TransientRetrySameChannel(err) => {
|
||||
self.finalize_result(Err(err)).await
|
||||
}
|
||||
TierConfigReloadOutcome::Terminal(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1677,9 +1662,6 @@ impl PeerRestClient {
|
||||
Err(err) => return tier_config_reload_connection_outcome(err),
|
||||
};
|
||||
let mut request = Request::new(LoadTransitionTierConfigRequest {});
|
||||
if let Err(err) = set_tonic_mutation_body_digest(&mut request) {
|
||||
return TierConfigReloadOutcome::Terminal(Error::other(err));
|
||||
}
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
|
||||
let response = match client.load_transition_tier_config(request).await {
|
||||
@@ -1720,37 +1702,39 @@ fn tier_config_reload_connection_outcome(err: Error) -> TierConfigReloadOutcome
|
||||
}
|
||||
|
||||
fn is_tier_config_reload_connection_failure(err: &Error) -> bool {
|
||||
let message = err.to_string();
|
||||
// A bare "unavailable" is only trusted inside the local dial-failure
|
||||
// wrapper from `get_client`, never in application text.
|
||||
let message = err.to_string().to_ascii_lowercase();
|
||||
if message
|
||||
.to_ascii_lowercase()
|
||||
.split_once("can not get client, err:")
|
||||
.is_some_and(|(_, local_error)| local_error.contains("unavailable"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
message_has_network_needle(&message)
|
||||
[
|
||||
"temporarily offline",
|
||||
"transport error",
|
||||
"error trying to connect",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"connection closed",
|
||||
"connection aborted",
|
||||
"broken pipe",
|
||||
"not connected",
|
||||
"unexpected eof",
|
||||
"timed out",
|
||||
"deadline has elapsed",
|
||||
"tcp connect error",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| message.contains(needle))
|
||||
}
|
||||
|
||||
/// Classifies a reload the peer answered but refused to apply.
|
||||
///
|
||||
/// The peer replied, so the channel is healthy and only the remote apply
|
||||
/// failed. Those failures are transient by nature: the reload reads the tier
|
||||
/// mutation intents and takes the distributed tier-config lock, both of which
|
||||
/// fail while any other node is restarting or while the lock quorum is briefly
|
||||
/// disturbed. Retiring the worker on the first such rejection leaves that peer
|
||||
/// pinned to the old configuration with nothing left to heal it, so it answers
|
||||
/// `TierNotFound` for a tier the rest of the cluster already committed until a
|
||||
/// second admin mutation happens to spawn a fresh worker.
|
||||
///
|
||||
/// Convergence is the whole point of this path, so a rejection is retried on
|
||||
/// the same channel. The worker's exponential backoff caps the cost at one
|
||||
/// reload every `TIER_CONFIG_RELOAD_RETRY_CAP`, and `Terminal` stays reachable
|
||||
/// for transport and gRPC status failures, which is where a genuinely
|
||||
/// unrecoverable peer surfaces.
|
||||
fn tier_config_reload_remote_failure(error_info: Option<String>) -> TierConfigReloadOutcome {
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info.unwrap_or_default()))
|
||||
let error_info = error_info.unwrap_or_default();
|
||||
if matches!(error_info.as_str(), "errServerNotInitialized" | "ServerNotInitialized") {
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info))
|
||||
} else {
|
||||
TierConfigReloadOutcome::Terminal(Error::other(error_info))
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadOutcome {
|
||||
@@ -1760,14 +1744,6 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
|
||||
TierConfigReloadOutcome::TransientReconnect(status.into())
|
||||
} else if status.code() == Code::Unknown && status.message().starts_with("Service was not ready:") {
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(status.into())
|
||||
} else if status.code() == Code::Unknown
|
||||
&& is_tier_config_reload_connection_failure(&Error::other(status.message().to_string()))
|
||||
{
|
||||
// tonic reports a connection dropped mid-call as `Unknown` carrying the
|
||||
// transport error text rather than as `Unavailable`, which is what a peer
|
||||
// restarting under an active mutation produces. Reconnect and retry, so
|
||||
// the restart does not permanently retire this peer's reload worker.
|
||||
TierConfigReloadOutcome::TransientReconnect(status.into())
|
||||
} else {
|
||||
TierConfigReloadOutcome::Terminal(status.into())
|
||||
}
|
||||
@@ -2175,126 +2151,6 @@ mod tests {
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::NotImplemented));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_uses_typed_status_code() {
|
||||
// The one code that means "nothing is answering on this channel".
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::from(tonic::Status::unavailable(
|
||||
"storage layer is not initialized"
|
||||
))));
|
||||
// Application statuses from a live peer must not mark it offline,
|
||||
// even when their message contains transport-sounding words.
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(tonic::Status::internal(
|
||||
"failed to reload metadata for bucket \"unavailable-logs\""
|
||||
))));
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(tonic::Status::unauthenticated(
|
||||
"No valid auth token"
|
||||
))));
|
||||
// A request-budget expiry answered by a live peer is an application
|
||||
// outcome, not a transport failure.
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(tonic::Status::deadline_exceeded(
|
||||
"heal control request expired"
|
||||
))));
|
||||
// Unknown is the transport's escape hatch for a cause it could not
|
||||
// map, and our handlers never return it, so there the text decides.
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::from(tonic::Status::unknown(
|
||||
"Service was not ready: transport error"
|
||||
))));
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(tonic::Status::unknown(
|
||||
"peer response unknown"
|
||||
))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_ignores_transport_words_in_application_statuses() {
|
||||
// The reason classification reads the code rather than the text: a
|
||||
// peer that answers is reachable, even when what it says describes a
|
||||
// connection failure of its own. A handler interpolating a local
|
||||
// io::Error into Status::internal, or relaying trouble with its own
|
||||
// downstream, must not cost us the channel to a healthy peer.
|
||||
for status in [
|
||||
tonic::Status::internal("connection refused while dialing downstream backend"),
|
||||
tonic::Status::internal("write failed: broken pipe"),
|
||||
tonic::Status::unauthenticated("connection reset while validating token"),
|
||||
tonic::Status::failed_precondition("scanner lease timed out"),
|
||||
tonic::Status::deadline_exceeded("heal control request timed out"),
|
||||
] {
|
||||
let rendered = status.to_string();
|
||||
assert!(
|
||||
!PeerRestClient::is_network_like_error(&Error::from(status)),
|
||||
"an answered application status must not mark the peer offline: {rendered}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_keeps_slow_peers_online() {
|
||||
// The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s)
|
||||
// surfaces as Cancelled "Timeout expired" carrying the transport
|
||||
// cause as its source. A peer that is merely slow must stay online:
|
||||
// gating it would spend a full recovery cycle fast-failing every RPC
|
||||
// to a host that is still answering, turning load into a partition.
|
||||
let timeout_status = tonic::Status::cancelled("Timeout expired");
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(timeout_status)));
|
||||
|
||||
let sourced = tonic::Status::from_error(Box::new(std::io::Error::other("Timeout expired")));
|
||||
assert!(
|
||||
std::error::Error::source(&sourced).is_some(),
|
||||
"the transport builds this status through Status::from_error, which attaches the cause"
|
||||
);
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::from(sourced)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rpc_status_errors_keep_their_rendering_and_hide_peer_metadata() {
|
||||
let err = Error::from(tonic::Status::unavailable("peer gone"));
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
"Io error: code: 'The service is currently unavailable', message: \"peer gone\""
|
||||
);
|
||||
|
||||
// tonic's own Debug prints the MetadataMap, i.e. every response header
|
||||
// the peer sent; those must not reach a log through this error.
|
||||
let mut status = tonic::Status::unavailable("peer gone");
|
||||
status
|
||||
.metadata_mut()
|
||||
.insert("authorization", "Bearer secret".parse().expect("valid header value"));
|
||||
let rendered = format!("{:?}", Error::from(status));
|
||||
assert!(!rendered.contains("Bearer secret"), "{rendered}");
|
||||
assert!(!rendered.contains("MetadataMap"), "{rendered}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_ignores_application_text_containing_unavailable() {
|
||||
// Regression: a bare "unavailable" needle used to match application
|
||||
// strings like these and take a healthy peer offline.
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::other(
|
||||
"peer replication statistics provider is unavailable"
|
||||
)));
|
||||
assert!(!PeerRestClient::is_network_like_error(&Error::other(
|
||||
"bucket \"unavailable-logs\" not found"
|
||||
)));
|
||||
// Anchored renderings of a flattened Unavailable status still match:
|
||||
// tonic >= 0.14 form ...
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::other(
|
||||
"peer tier mutation commit RPC failed: code: 'The service is currently unavailable', message: \"peer gone\""
|
||||
)));
|
||||
// ... which is only anchored as long as tonic renders Unavailable this
|
||||
// way. A tonic bump that reworded it leaves the typed path correct but
|
||||
// this needle stale, so pin the coupling rather than discover it in a
|
||||
// partition.
|
||||
assert!(
|
||||
tonic::Status::unavailable("peer gone")
|
||||
.to_string()
|
||||
.to_ascii_lowercase()
|
||||
.contains("code: 'the service is currently unavailable'"),
|
||||
"tonic reworded Code::Unavailable; update the anchored needle"
|
||||
);
|
||||
// ... and the tonic <= 0.13 form peers may relay in error_info.
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::other(
|
||||
"peer tier mutation commit RPC failed: status: Unavailable, message: \"peer gone\""
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_config_reload_outcome_keeps_tonic_and_remote_errors_typed() {
|
||||
assert!(matches!(
|
||||
@@ -2321,12 +2177,9 @@ mod tests {
|
||||
tier_config_reload_status_outcome(tonic::Status::cancelled("request cancelled")),
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
// A peer that answered and then refused the apply is retried rather than
|
||||
// retired: the channel is healthy, so the rejection reflects remote state
|
||||
// that the next attempt can find healed.
|
||||
assert!(matches!(
|
||||
tier_config_reload_remote_failure(Some("backend unavailable".to_string())),
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(_)
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
tier_config_reload_remote_failure(Some("errServerNotInitialized".to_string())),
|
||||
@@ -2340,58 +2193,6 @@ mod tests {
|
||||
tier_config_reload_connection_outcome(Error::other("can not get client, err: connection unavailable")),
|
||||
TierConfigReloadOutcome::TransientReconnect(_)
|
||||
));
|
||||
// The bare word is trusted only to the right of the dial-failure
|
||||
// prefix, not anywhere in the message.
|
||||
assert!(matches!(
|
||||
tier_config_reload_connection_outcome(Error::other(
|
||||
"bucket unavailable-logs rejected it, then: can not get client, err: some other reason"
|
||||
)),
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
}
|
||||
|
||||
/// A tier mutation issued while another node restarts must still converge on
|
||||
/// the nodes that stayed up. Those peers answer the reload RPC and reject the
|
||||
/// apply, because reloading reads the tier mutation intents and takes the
|
||||
/// distributed tier-config lock while the lock quorum is still disturbed.
|
||||
/// Classifying those rejections as terminal retired the reload worker on its
|
||||
/// first attempt and pinned the peer to the previous configuration, so it
|
||||
/// served `TierNotFound` for an already-committed tier until an unrelated
|
||||
/// second admin mutation spawned a new worker.
|
||||
#[test]
|
||||
fn tier_config_reload_retries_peers_that_reject_the_apply_mid_restart() {
|
||||
for error_info in [
|
||||
"Lock acquisition timeout for resource '.rustfs.sys/config/tier-config.bin.lock' after 5s",
|
||||
"Resource '.rustfs.sys/config/tier-config.bin.lock' is already locked by node-3",
|
||||
"Internal error: release RPC failed: transport error",
|
||||
"save_config_with_opts: err: PreconditionFailed",
|
||||
"erasure read quorum",
|
||||
] {
|
||||
assert!(
|
||||
matches!(
|
||||
tier_config_reload_remote_failure(Some(error_info.to_string())),
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(_)
|
||||
),
|
||||
"a peer that rejected the apply must stay retryable so it converges: {error_info}"
|
||||
);
|
||||
}
|
||||
|
||||
// An absent error message is still a rejection, not a reason to stop.
|
||||
assert!(matches!(
|
||||
tier_config_reload_remote_failure(None),
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(_)
|
||||
));
|
||||
|
||||
// tonic surfaces a connection dropped mid-call as `Unknown`, not `Unavailable`.
|
||||
assert!(matches!(
|
||||
tier_config_reload_status_outcome(tonic::Status::unknown("transport error")),
|
||||
TierConfigReloadOutcome::TransientReconnect(_)
|
||||
));
|
||||
// An `Unknown` that is not transport-shaped stays terminal.
|
||||
assert!(matches!(
|
||||
tier_config_reload_status_outcome(tonic::Status::unknown("peer response unknown")),
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2694,39 +2495,6 @@ mod tests {
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_app_errors_mentioning_unavailable() {
|
||||
// Regression: application error text containing "unavailable" (a
|
||||
// remote error_info payload, or a bucket named "unavailable-logs" in
|
||||
// a typed application status) must not take a healthy peer offline.
|
||||
let client = test_peer_client();
|
||||
let err = client
|
||||
.finalize_result::<()>(Err(Error::other("peer replication statistics provider is unavailable")))
|
||||
.await
|
||||
.expect_err("application error should still be returned");
|
||||
assert!(err.to_string().contains("provider is unavailable"));
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
|
||||
let err = client
|
||||
.finalize_result::<()>(Err(Error::from(tonic::Status::internal(
|
||||
"failed to reload metadata for bucket \"unavailable-logs\"",
|
||||
))))
|
||||
.await
|
||||
.expect_err("application status should still be returned");
|
||||
assert!(err.to_string().contains("unavailable-logs"));
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_marks_offline_for_typed_unavailable_status() {
|
||||
let client = test_peer_client();
|
||||
client
|
||||
.finalize_result::<()>(Err(Error::from(tonic::Status::unavailable("storage layer is not initialized"))))
|
||||
.await
|
||||
.expect_err("network error should still be returned");
|
||||
assert!(client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn peer_rest_recovery_probe_logs_keep_request_id_span_context() {
|
||||
let logs = CapturedLogs::default();
|
||||
|
||||
@@ -16,7 +16,6 @@ use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
|
||||
@@ -90,22 +89,6 @@ fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Erro
|
||||
reduce_write_quorum_errs(per_pool_errs, BUCKET_OP_IGNORED_ERRS, pool_write_quorum(per_pool_errs.len()))
|
||||
}
|
||||
|
||||
fn resolve_heal_bucket_mode(opts: &mut HealOpts, pool_errs: &[Option<Error>]) -> Result<()> {
|
||||
if opts.recreate {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(err) = pool_errs
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|err| **err != Error::DiskNotFound && **err != Error::VolumeNotFound)
|
||||
{
|
||||
return Err(err.clone());
|
||||
}
|
||||
opts.remove = is_all_buckets_not_found(pool_errs);
|
||||
opts.recreate = !opts.remove;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait PeerS3Client: Debug + Sync + Send + 'static {
|
||||
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
|
||||
@@ -176,7 +159,10 @@ impl S3PeerSys {
|
||||
pool_errs.push(reduce_pool_write_quorum_errs(&per_pool_errs));
|
||||
}
|
||||
|
||||
resolve_heal_bucket_mode(&mut opts, &pool_errs)?;
|
||||
if !opts.recreate {
|
||||
opts.remove = is_all_buckets_not_found(&pool_errs);
|
||||
opts.recreate = !opts.remove;
|
||||
}
|
||||
|
||||
let mut futures = Vec::new();
|
||||
let heal_bucket_results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.clients.len()]));
|
||||
@@ -931,11 +917,10 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
|| async {
|
||||
let options: String = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(HealBucketRequest {
|
||||
let request = Request::new(HealBucketRequest {
|
||||
bucket: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let response = client.heal_bucket(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return if let Some(err) = response.error {
|
||||
@@ -988,11 +973,10 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
|| async {
|
||||
let options = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MakeBucketRequest {
|
||||
let request = Request::new(MakeBucketRequest {
|
||||
name: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let response = client.make_bucket(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
@@ -1043,11 +1027,10 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
let options = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
|
||||
let mut request = Request::new(DeleteBucketRequest {
|
||||
let request = Request::new(DeleteBucketRequest {
|
||||
bucket: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let response = client.delete_bucket(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return if let Some(err) = response.error {
|
||||
@@ -1635,30 +1618,6 @@ mod tests {
|
||||
assert_eq!(err, Error::VolumeExists);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_bucket_mode_fails_closed_on_incomplete_topology() {
|
||||
let mut opts = HealOpts::default();
|
||||
assert_eq!(
|
||||
resolve_heal_bucket_mode(&mut opts, &[Some(Error::ErasureWriteQuorum)]),
|
||||
Err(Error::ErasureWriteQuorum)
|
||||
);
|
||||
assert!(!opts.recreate);
|
||||
assert!(!opts.remove);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_bucket_mode_distinguishes_deleted_and_partial_buckets() {
|
||||
let mut deleted = HealOpts::default();
|
||||
resolve_heal_bucket_mode(&mut deleted, &[Some(Error::VolumeNotFound)]).unwrap();
|
||||
assert!(deleted.remove);
|
||||
assert!(!deleted.recreate);
|
||||
|
||||
let mut partial = HealOpts::default();
|
||||
resolve_heal_bucket_mode(&mut partial, &[None, Some(Error::VolumeNotFound)]).unwrap();
|
||||
assert!(!partial.remove);
|
||||
assert!(partial.recreate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_make_bucket_reduces_quorum_by_pool_participants() {
|
||||
let peer_sys = S3PeerSys {
|
||||
|
||||
@@ -25,7 +25,7 @@ use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
disk_store::{
|
||||
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
|
||||
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
|
||||
@@ -50,9 +50,8 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
|
||||
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
|
||||
RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest,
|
||||
WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -101,18 +100,6 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
|
||||
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
|
||||
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
|
||||
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
|
||||
pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
|
||||
|
||||
fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.protocol_version != SNAPSHOT_LEASE_PROTOCOL_VERSION {
|
||||
return Err(Error::other("remote snapshot lease protocol is incompatible"));
|
||||
}
|
||||
SnapshotLeaseToken::from_slice(&response.token)
|
||||
}
|
||||
|
||||
/// Bind a mutating disk RPC to its canonical body: the digest lands in the request metadata, and
|
||||
/// the signing interceptor folds it (plus a replay-protected nonce) into the v2 signature scope
|
||||
@@ -1797,81 +1784,6 @@ impl DiskAPI for RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SnapshotLeaseRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
|
||||
.map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "acquire_snapshot_lease")?;
|
||||
let response = client.acquire_snapshot_lease(request).await?.into_inner();
|
||||
snapshot_lease_token_from_response(response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SnapshotLeaseRenewRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
|
||||
.map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "renew_snapshot_lease")?;
|
||||
let response = client.renew_snapshot_lease(request).await?.into_inner();
|
||||
snapshot_lease_token_from_response(response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SnapshotLeaseReleaseRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_release_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "release_snapshot_lease")?;
|
||||
let response = client.release_snapshot_lease(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
trace!(
|
||||
@@ -3018,34 +2930,6 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
#[test]
|
||||
fn snapshot_lease_response_requires_current_protocol_and_valid_token() {
|
||||
let token = SnapshotLeaseToken::new();
|
||||
let response = SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
|
||||
error: None,
|
||||
};
|
||||
assert_eq!(snapshot_lease_token_from_response(response).unwrap(), token);
|
||||
|
||||
let incompatible = SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION + 1,
|
||||
error: None,
|
||||
};
|
||||
assert!(snapshot_lease_token_from_response(incompatible).is_err());
|
||||
|
||||
let malformed = SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: Bytes::from_static(b"not-a-uuid"),
|
||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
|
||||
error: None,
|
||||
};
|
||||
assert!(snapshot_lease_token_from_response(malformed).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_volumes_decode_rejects_a_malformed_entry() {
|
||||
let valid = serde_json::to_string(&VolumeInfo {
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use rustfs_lock::{
|
||||
@@ -314,11 +313,10 @@ impl LockClient for RemoteClient {
|
||||
info!("remote acquire_exclusive for {}", request.resource);
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = match self.execute_rpc("lock", &resource_summary, client.lock(req)).await {
|
||||
Ok(resp) => resp.into_inner(),
|
||||
@@ -349,7 +347,7 @@ impl LockClient for RemoteClient {
|
||||
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(requests);
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
let req = Request::new(BatchGenerallyLockRequest {
|
||||
args: requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
@@ -357,7 +355,6 @@ impl LockClient for RemoteClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc("lock_batch", &resource_summary, client.lock_batch(req))
|
||||
@@ -398,8 +395,7 @@ impl LockClient for RemoteClient {
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = unlock_request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
let req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
let resp = self
|
||||
.execute_rpc("release", &resource_summary, client.un_lock(req))
|
||||
.await?
|
||||
@@ -418,7 +414,7 @@ impl LockClient for RemoteClient {
|
||||
let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(&unlock_requests);
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
let req = Request::new(BatchGenerallyLockRequest {
|
||||
args: unlock_requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
@@ -426,7 +422,6 @@ impl LockClient for RemoteClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = self
|
||||
.execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
|
||||
@@ -445,11 +440,10 @@ impl LockClient for RemoteClient {
|
||||
let refresh_request = Self::create_unlock_request(lock_id);
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = refresh_request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&refresh_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("refresh", &resource_summary, client.refresh(req))
|
||||
.await?
|
||||
@@ -465,11 +459,10 @@ impl LockClient for RemoteClient {
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = force_request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&force_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
|
||||
.await?
|
||||
@@ -490,11 +483,10 @@ impl LockClient for RemoteClient {
|
||||
let mut client = self.get_client().await?;
|
||||
|
||||
// Try to acquire a very short-lived lock to test availability
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
// Try exclusive lock first with very short timeout
|
||||
let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
|
||||
@@ -505,11 +497,10 @@ impl LockClient for RemoteClient {
|
||||
if resp.success {
|
||||
// If we successfully acquired the lock, the resource was free.
|
||||
// Immediately release it on a best-effort basis.
|
||||
let mut release_req = Request::new(GenerallyLockRequest {
|
||||
let release_req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut release_req)?;
|
||||
let _ = self
|
||||
.execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
|
||||
.await;
|
||||
|
||||
@@ -2543,7 +2543,6 @@ mod tests {
|
||||
data_dir: None,
|
||||
delete_marker: false,
|
||||
transitioned_object: Default::default(),
|
||||
transition_version_state: Default::default(),
|
||||
restore_ongoing: false,
|
||||
restore_expires: None,
|
||||
user_tags: Arc::new(String::new()),
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::{
|
||||
CheckPartsResp, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint,
|
||||
Error, FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result,
|
||||
SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error,
|
||||
FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
health_state::{
|
||||
RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
|
||||
get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
|
||||
@@ -1349,38 +1349,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.acquire_snapshot_lease(volume, path).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.release_snapshot_lease(volume, path, token).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.renew_snapshot_lease(volume, path, token).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.delete_data_dir(volume, path, opts).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.write_metadata(org_volume, volume, path, fi).await },
|
||||
|
||||
@@ -337,54 +337,9 @@ impl From<DiskError> for std::io::Error {
|
||||
}
|
||||
}
|
||||
|
||||
/// The single in-band representation of a failed internode RPC: it carries the
|
||||
/// typed `tonic::Status` so failure classifiers can read the gRPC code instead
|
||||
/// of substring-matching the rendered message (see `is_network_like_status`).
|
||||
/// Both `DiskError` and `StorageError` wrap statuses in this type, so one
|
||||
/// downcast recovers the status regardless of which error the status was
|
||||
/// converted into first.
|
||||
pub(crate) struct RpcStatusError(tonic::Status);
|
||||
|
||||
impl RpcStatusError {
|
||||
pub(crate) fn status(&self) -> &tonic::Status {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<tonic::Status> for RpcStatusError {
|
||||
fn from(status: tonic::Status) -> Self {
|
||||
Self(status)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for RpcStatusError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
std::fmt::Display::fmt(&self.0, f)
|
||||
}
|
||||
}
|
||||
|
||||
/// `tonic::Status`'s own `Debug` prints its `MetadataMap`, i.e. every response
|
||||
/// header and trailer the peer sent. Those are remote-controlled and can carry
|
||||
/// credentials injected by a proxy in front of the peer, so keep them out of
|
||||
/// anything that reaches a log.
|
||||
impl std::fmt::Debug for RpcStatusError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RpcStatusError")
|
||||
.field("code", &self.0.code())
|
||||
.field("message", &self.0.message())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl StdError for RpcStatusError {
|
||||
fn source(&self) -> Option<&(dyn StdError + 'static)> {
|
||||
Some(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<tonic::Status> for DiskError {
|
||||
fn from(e: tonic::Status) -> Self {
|
||||
DiskError::Io(io::Error::other(RpcStatusError(e)))
|
||||
DiskError::other(e.message().to_string())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -72,39 +72,6 @@ pub type DiskStore = Arc<Disk>;
|
||||
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
|
||||
pub type FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SnapshotLeaseToken(Uuid);
|
||||
|
||||
impl SnapshotLeaseToken {
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::new_v4())
|
||||
}
|
||||
|
||||
pub fn from_slice(bytes: &[u8]) -> Result<Self> {
|
||||
let uuid = Uuid::from_slice(bytes).map_err(|_| Error::other("invalid snapshot lease token"))?;
|
||||
if uuid.is_nil() {
|
||||
return Err(Error::other("invalid snapshot lease token"));
|
||||
}
|
||||
Ok(Self(uuid))
|
||||
}
|
||||
|
||||
pub fn as_bytes(&self) -> &[u8; 16] {
|
||||
self.0.as_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SnapshotLeaseToken {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum DataDirDeleteStatus {
|
||||
Deleted,
|
||||
Deferred,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PartTransactionAction {
|
||||
Commit,
|
||||
@@ -282,34 +249,6 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.acquire_snapshot_lease(volume, path).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.acquire_snapshot_lease(volume, path).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.release_snapshot_lease(volume, path, token).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.release_snapshot_lease(volume, path, token).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.renew_snapshot_lease(volume, path, token).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.renew_snapshot_lease(volume, path, token).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.delete_data_dir(volume, path, opts).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.delete_data_dir(volume, path, opts).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
match self {
|
||||
@@ -707,19 +646,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
) -> Result<()>;
|
||||
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>>;
|
||||
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()>;
|
||||
async fn acquire_snapshot_lease(&self, _volume: &str, _path: &str) -> Result<SnapshotLeaseToken> {
|
||||
Err(Error::other("snapshot leases are not supported by this disk"))
|
||||
}
|
||||
async fn release_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<()> {
|
||||
Err(Error::other("snapshot leases are not supported by this disk"))
|
||||
}
|
||||
async fn renew_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
Err(Error::other("snapshot leases are not supported by this disk"))
|
||||
}
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
self.delete(volume, path, opts).await?;
|
||||
Ok(DataDirDeleteStatus::Deleted)
|
||||
}
|
||||
async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
|
||||
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()>;
|
||||
async fn read_version(
|
||||
|
||||
@@ -818,11 +818,7 @@ impl From<s3s::xml::DeError> for Error {
|
||||
|
||||
impl From<tonic::Status> for Error {
|
||||
fn from(e: tonic::Status) -> Self {
|
||||
// Keep the typed status as the io::Error payload instead of a
|
||||
// flattened string so RPC failure classifiers can read the gRPC code
|
||||
// via downcast (see `PeerRestClient::is_network_like_error`).
|
||||
// `RpcStatusError` renders exactly as `e.to_string()` did.
|
||||
Error::other(crate::disk::error::RpcStatusError::from(e))
|
||||
Error::other(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -180,7 +180,6 @@ pub struct ObjectInfo {
|
||||
pub data_dir: Option<Uuid>,
|
||||
pub delete_marker: bool,
|
||||
pub transitioned_object: TransitionedObject,
|
||||
pub transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
pub restore_ongoing: bool,
|
||||
pub restore_expires: Option<OffsetDateTime>,
|
||||
pub user_tags: Arc<String>,
|
||||
@@ -221,7 +220,6 @@ impl Clone for ObjectInfo {
|
||||
data_dir: self.data_dir,
|
||||
delete_marker: self.delete_marker,
|
||||
transitioned_object: self.transitioned_object.clone(),
|
||||
transition_version_state: self.transition_version_state,
|
||||
restore_ongoing: self.restore_ongoing,
|
||||
restore_expires: self.restore_expires,
|
||||
user_tags: self.user_tags.clone(),
|
||||
@@ -466,11 +464,11 @@ impl ObjectInfo {
|
||||
|
||||
let transitioned_object = TransitionedObject {
|
||||
name: fi.transitioned_objname.clone(),
|
||||
version_id: fi
|
||||
.transition_version
|
||||
.clone()
|
||||
.or_else(|| fi.transition_version_id.map(|version_id| version_id.to_string()))
|
||||
.unwrap_or_default(),
|
||||
version_id: if let Some(transition_version_id) = fi.transition_version_id {
|
||||
transition_version_id.to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
status: fi.transition_status.clone(),
|
||||
free_version: fi.tier_free_version(),
|
||||
tier: fi.transition_tier.clone(),
|
||||
@@ -539,7 +537,6 @@ impl ObjectInfo {
|
||||
inlined,
|
||||
user_defined: Arc::new(metadata),
|
||||
transitioned_object,
|
||||
transition_version_state: fi.transition_version_state,
|
||||
checksum: fi.checksum.clone(),
|
||||
storage_class,
|
||||
restore_ongoing,
|
||||
|
||||
@@ -207,6 +207,10 @@ pub(crate) async fn ensure_boot_time() {
|
||||
GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_init_time() -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
rustfs_common::get_global_init_time().await
|
||||
}
|
||||
|
||||
pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option<u64> {
|
||||
if is_erasure_sd().await {
|
||||
None
|
||||
|
||||
@@ -71,7 +71,6 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
|
||||
MadminScannerMetrics {
|
||||
collected_at: metrics.collected_at,
|
||||
current_cycle: metrics.current_cycle,
|
||||
current_cycle_active: Some(metrics.current_cycle_active),
|
||||
current_started: metrics.current_started,
|
||||
cycles_completed_at: metrics.cycles_completed_at,
|
||||
ongoing_buckets: metrics.ongoing_buckets,
|
||||
@@ -399,7 +398,10 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
|
||||
|
||||
if types.contains(&MetricType::SCANNER) {
|
||||
debug!("start get scanner metrics");
|
||||
let metrics = global_metrics().report().await;
|
||||
let mut metrics = global_metrics().report().await;
|
||||
if let Some(init_time) = runtime_sources::scanner_init_time().await {
|
||||
metrics.current_started = init_time;
|
||||
}
|
||||
real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics));
|
||||
}
|
||||
|
||||
@@ -538,9 +540,7 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use rustfs_common::metrics::CurrentCycle;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
@@ -588,10 +588,7 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_partial_source_status() {
|
||||
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
|
||||
current_cycle_active: true,
|
||||
current_started,
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
|
||||
@@ -601,8 +598,6 @@ mod test {
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
let usage = scanner
|
||||
@@ -613,39 +608,6 @@ mod test {
|
||||
assert_eq!(usage.cycles, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
|
||||
let previous_init_time = *rustfs_common::globals::GLOBAL_INIT_TIME.read().await;
|
||||
let previous_cycle = global_metrics().get_cycle().await;
|
||||
let init_time = Utc::now() - chrono::Duration::hours(1);
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
||||
let cycle = CurrentCycle {
|
||||
current: 0,
|
||||
next: 1,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_start = global_metrics().start_scan_cycle_work_with_cycle(cycle).await;
|
||||
|
||||
let realtime = collect_local_metrics(MetricType::SCANNER, &CollectMetricsOpts::default()).await;
|
||||
|
||||
global_metrics()
|
||||
.finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
|
||||
.await;
|
||||
global_metrics().set_cycle(previous_cycle).await;
|
||||
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
|
||||
|
||||
let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
|
||||
let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
|
||||
let mut aggregated = RealtimeMetrics::default();
|
||||
aggregated.merge(decoded);
|
||||
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, cycle_started);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_pacing_pressure() {
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
|
||||
|
||||
@@ -1344,11 +1344,8 @@ async fn run_tier_config_reload_worker<F, Fut>(
|
||||
}
|
||||
TierConfigReloadFinish::Pending => retry_attempt = 0,
|
||||
},
|
||||
TierConfigReloadOutcome::Terminal(err) => match sys.finish_tier_config_reload_worker(&host) {
|
||||
TierConfigReloadOutcome::Terminal(_) => match sys.finish_tier_config_reload_worker(&host) {
|
||||
TierConfigReloadFinish::Completed => {
|
||||
// This peer keeps the previous tier configuration for good, so record
|
||||
// why. Dropping the error here hides the only evidence of a divergent
|
||||
// node behind an outcome label that cannot be acted on.
|
||||
warn!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -1356,7 +1353,6 @@ async fn run_tier_config_reload_worker<F, Fut>(
|
||||
action = "reload_transition_tier_config",
|
||||
host,
|
||||
outcome = "terminal",
|
||||
error = ?err,
|
||||
"tier configuration reload stopped after a terminal outcome"
|
||||
);
|
||||
return;
|
||||
|
||||
@@ -931,10 +931,7 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
|
||||
status: fi.transition_status.clone(),
|
||||
tier: fi.transition_tier.clone(),
|
||||
remote_object: fi.transitioned_objname.clone(),
|
||||
remote_version_id: fi
|
||||
.transition_version
|
||||
.clone()
|
||||
.or_else(|| fi.transition_version_id.map(|id| id.to_string())),
|
||||
remote_version_id: fi.transition_version_id.map(|id| id.to_string()),
|
||||
free_version_count,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9274,8 +9274,7 @@ mod tests {
|
||||
version_id: "v1".to_string(),
|
||||
tier_name: "COLD-A".to_string(),
|
||||
backend_identity: Some(current_identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
journal_store
|
||||
.insert_config_object(
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
@@ -46,19 +45,6 @@ const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
|
||||
if remote_version.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let generation = remote_version
|
||||
.parse::<i64>()
|
||||
.map_err(|_| Error::new(ErrorKind::InvalidData, "GCS remote version is not a valid generation"))?;
|
||||
if generation <= 0 {
|
||||
return Err(Error::new(ErrorKind::InvalidData, "GCS remote version generation must be positive"));
|
||||
}
|
||||
Ok(Some(generation))
|
||||
}
|
||||
|
||||
pub struct WarmBackendGCS {
|
||||
pub client: Arc<Storage>,
|
||||
pub control: Arc<StorageControl>,
|
||||
@@ -119,10 +105,6 @@ impl WarmBackendGCS {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendGCS {
|
||||
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
|
||||
parse_generation(remote_version_id).map(|_| ())
|
||||
}
|
||||
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
@@ -153,9 +135,6 @@ impl WarmBackend for WarmBackendGCS {
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
}
|
||||
|
||||
// Honor the requested byte range so Range GETs on tiered objects return the exact
|
||||
// interval instead of the whole object (matches the s3/s3sdk/rustfs warm backends).
|
||||
@@ -185,15 +164,13 @@ impl WarmBackend for WarmBackendGCS {
|
||||
// gRPC v2 DeleteObject requires the bucket in resource-name form. Without this the
|
||||
// deleted tiered object was never removed from GCS (empty impl returned Ok), leaking
|
||||
// remote data forever.
|
||||
let mut req = self
|
||||
.control
|
||||
self.control
|
||||
.delete_object()
|
||||
.set_bucket(format!("projects/_/buckets/{}", self.bucket))
|
||||
.set_object(self.get_dest(object));
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
}
|
||||
req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
.set_object(self.get_dest(object))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -214,30 +191,6 @@ impl WarmBackend for WarmBackendGCS {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_generation;
|
||||
use std::io::ErrorKind;
|
||||
|
||||
#[test]
|
||||
fn generation_parser_preserves_exact_numeric_versions() {
|
||||
assert_eq!(parse_generation("").expect("empty generation means no version condition"), None);
|
||||
assert_eq!(parse_generation("1").expect("minimum generation should parse"), Some(1));
|
||||
assert_eq!(
|
||||
parse_generation(&i64::MAX.to_string()).expect("maximum generation should parse"),
|
||||
Some(i64::MAX)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generation_parser_rejects_unknown_or_non_positive_versions() {
|
||||
for value in ["unknown", "1.0", "-1", "0", "9223372036854775808"] {
|
||||
let err = parse_generation(value).expect_err("unknown generation must fail closed");
|
||||
assert_eq!(err.kind(), ErrorKind::InvalidData, "{value}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
|
||||
if err == nil {
|
||||
return nil
|
||||
|
||||
@@ -46,10 +46,9 @@ use crate::diagnostics::get::{
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
||||
use crate::disk::{
|
||||
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
|
||||
PartTransactionAction, part_transaction_path,
|
||||
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
||||
part_transaction_path,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
@@ -548,8 +547,6 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo
|
||||
&& left.transitioned_objname == right.transitioned_objname
|
||||
&& left.transition_tier == right.transition_tier
|
||||
&& left.transition_version_id == right.transition_version_id
|
||||
&& left.transition_version == right.transition_version
|
||||
&& left.transition_version_state == right.transition_version_state
|
||||
&& left.expire_restored == right.expire_restored
|
||||
&& left.size == right.size
|
||||
&& left.mod_time == right.mod_time
|
||||
@@ -2988,48 +2985,29 @@ impl SetDisks {
|
||||
Self::rename_fanout_barrier(&object_for_fault, idx, rename_fanout_barrier_phase::CLEANUP).await;
|
||||
|
||||
if let Some(err) = Self::cleanup_injected_error(&object_for_fault, idx) {
|
||||
return (false, Some(err));
|
||||
return Some(err);
|
||||
}
|
||||
if let Some(disk) = disk {
|
||||
match disk
|
||||
.delete_data_dir(
|
||||
&bucket,
|
||||
&file_path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DataDirDeleteStatus::Deleted) => (false, None),
|
||||
Ok(DataDirDeleteStatus::Deferred) => (true, None),
|
||||
Err(err) => (false, Some(err)),
|
||||
}
|
||||
disk.delete(
|
||||
&bucket,
|
||||
&file_path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
} else {
|
||||
// `None` slot: ignored placeholder. It is not `attempted`, so
|
||||
// classification excludes it from residue regardless.
|
||||
(false, Some(DiskError::DiskNotFound))
|
||||
Some(DiskError::DiskNotFound)
|
||||
}
|
||||
})
|
||||
});
|
||||
let mut deferred = 0usize;
|
||||
let errs: Vec<Option<DiskError>> = join_all(futures)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|result| match result {
|
||||
Ok((was_deferred, err)) => {
|
||||
deferred += usize::from(was_deferred);
|
||||
err
|
||||
}
|
||||
Err(join_err) => Some(DiskError::other(format!("old data dir cleanup task failed: {join_err}"))),
|
||||
})
|
||||
.collect();
|
||||
let errs: Vec<Option<DiskError>> = join_all(futures).await.into_iter().map(map_cleanup_join_result).collect();
|
||||
|
||||
let mut cleanup = classify_old_data_dir_cleanup(&errs, &attempted, write_quorum);
|
||||
cleanup.deferred = deferred;
|
||||
cleanup.reclaimed = cleanup.reclaimed.saturating_sub(deferred);
|
||||
cleanup
|
||||
classify_old_data_dir_cleanup(&errs, &attempted, write_quorum)
|
||||
}
|
||||
|
||||
/// Test-only fault-injection seam for the old-data-dir cleanup path
|
||||
@@ -3120,20 +3098,6 @@ impl SetDisks {
|
||||
|
||||
rustfs_io_metrics::record_old_data_dir_cleanup(c.attempted, c.reclaimed, c.unreclaimed_disks.len(), c.below_quorum);
|
||||
|
||||
if c.deferred > 0 {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_WRITE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
old_data_dir = %old_dir,
|
||||
deferred = c.deferred,
|
||||
state = "old_data_cleanup_deferred",
|
||||
"Old data directory cleanup deferred for active snapshot leases"
|
||||
);
|
||||
}
|
||||
|
||||
if actions.warn {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -3953,9 +3917,9 @@ impl SetDisks {
|
||||
/// * The set of referenced data dirs is the UNION of `get_data_dirs()` across
|
||||
/// every online disk's `xl.meta`, so a dir named by *any* replica is kept.
|
||||
/// * If a disk holds the object directory but its `xl.meta` is missing or
|
||||
/// unparsable, the object is treated as degraded and unmarked data dirs are
|
||||
/// never removed. A data dir carrying a committed delete-transaction marker
|
||||
/// remains reclaimable after a downgrade/re-upgrade cleanup interruption.
|
||||
/// unparsable, the object is treated as degraded and NOTHING is removed —
|
||||
/// the unreadable copy could be the only one naming a live data dir, and a
|
||||
/// heal must run first.
|
||||
/// * Only subdirectories whose names parse as a UUID are ever considered;
|
||||
/// removal is non-recursive-safe via a recursive delete of the full stray
|
||||
/// data-dir path only.
|
||||
@@ -3970,7 +3934,7 @@ impl SetDisks {
|
||||
// physical UUID subdirectories present on each disk. Abort on any degraded
|
||||
// copy so a healable object is never stripped of a referenced data dir.
|
||||
let mut referenced: HashSet<Uuid> = HashSet::new();
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<(Uuid, bool)>)> = Vec::new();
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<Uuid>)> = Vec::new();
|
||||
let mut healthy_metas = 0usize;
|
||||
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
@@ -4006,22 +3970,6 @@ impl SetDisks {
|
||||
// to the orphan-dir / dangling-object heal paths.
|
||||
continue;
|
||||
}
|
||||
let mut committed = Vec::with_capacity(physical.len());
|
||||
for dir in physical {
|
||||
let data_dir = format!("{object}/{dir}");
|
||||
let committed_delete = disk.list_dir("", bucket, &data_dir, 0).await.is_ok_and(|entries| {
|
||||
entries.iter().any(|entry| {
|
||||
entry
|
||||
.strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
|
||||
.is_some_and(|transaction| Uuid::parse_str(transaction).is_ok())
|
||||
})
|
||||
});
|
||||
committed.push((dir, committed_delete));
|
||||
}
|
||||
if committed.iter().all(|(_, committed_delete)| *committed_delete) {
|
||||
per_disk_dirs.push((i, committed));
|
||||
continue;
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
bucket, object,
|
||||
@@ -4058,16 +4006,22 @@ impl SetDisks {
|
||||
|
||||
healthy_metas += 1;
|
||||
if !physical.is_empty() {
|
||||
per_disk_dirs.push((i, physical.into_iter().map(|dir| (dir, false)).collect()));
|
||||
per_disk_dirs.push((i, physical));
|
||||
}
|
||||
}
|
||||
|
||||
// No healthy metadata anywhere: this is not a live object, so surplus dirs
|
||||
// (if any) belong to the dangling-object heal path, not here.
|
||||
if healthy_metas == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Phase 2: delete every physical data dir not referenced by the union.
|
||||
let mut removed = 0usize;
|
||||
for (i, physical) in per_disk_dirs {
|
||||
let Some(disk) = disks[i].as_ref() else { continue };
|
||||
for (dir, committed_delete) in physical {
|
||||
if referenced.contains(&dir) || (healthy_metas == 0 && !committed_delete) {
|
||||
for dir in physical {
|
||||
if referenced.contains(&dir) {
|
||||
continue;
|
||||
}
|
||||
let stray = format!("{object}/{dir}");
|
||||
@@ -4175,9 +4129,6 @@ pub(in crate::set_disk) struct OldDataDirCleanup {
|
||||
/// Number of attempted disks that returned `Ok` or a not-found variant
|
||||
/// (a missing dir == already reclaimed).
|
||||
pub reclaimed: usize,
|
||||
/// Number of attempted disks that retained the directory for an active
|
||||
/// snapshot lease and registered it for deletion after the final release.
|
||||
pub deferred: usize,
|
||||
/// Indices of attempted disks that failed with a non-ignored, non-not-found
|
||||
/// error (including task panic/cancel). This is the residue that actually
|
||||
/// leaks and drives the leak metric + heal enqueue.
|
||||
@@ -4240,7 +4191,6 @@ fn classify_old_data_dir_cleanup(errs: &[Option<DiskError>], attempted: &[bool],
|
||||
OldDataDirCleanup {
|
||||
attempted: attempted_count,
|
||||
reclaimed,
|
||||
deferred: 0,
|
||||
unreclaimed_disks,
|
||||
below_quorum,
|
||||
}
|
||||
@@ -5181,40 +5131,6 @@ mod tests {
|
||||
drop((disk1, disk2));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn commit_cleanup_reports_and_releases_deferred_snapshot_data_dirs() {
|
||||
let bucket = "cleanup-lease-bucket";
|
||||
let object = "cleanup-lease-object";
|
||||
let old_data_dir = "11111111-1111-1111-1111-111111111111";
|
||||
let committed_data_dir = "22222222-2222-2222-2222-222222222222";
|
||||
let data_dir_path = format!("{object}/{old_data_dir}");
|
||||
let shard_path = format!("{data_dir_path}/part.1");
|
||||
let (_dir1, disk1) = read_multiple_test_disk(bucket, &[(&shard_path, b"one".as_slice())]).await;
|
||||
let set = io_primitives_test_set(vec![Some(disk1.clone())], 0).await;
|
||||
let lease = disk1
|
||||
.acquire_snapshot_lease(bucket, &data_dir_path)
|
||||
.await
|
||||
.expect("snapshot lease should be acquired before cleanup");
|
||||
|
||||
let cleanup = set
|
||||
.commit_rename_data_dir(&[Some(disk1.clone())], bucket, object, old_data_dir, committed_data_dir, 1)
|
||||
.await;
|
||||
assert_eq!(cleanup.attempted, 1);
|
||||
assert_eq!(cleanup.reclaimed, 0);
|
||||
assert_eq!(cleanup.deferred, 1);
|
||||
assert!(cleanup.unreclaimed_disks.is_empty());
|
||||
disk1
|
||||
.read_all(bucket, &shard_path)
|
||||
.await
|
||||
.expect("deferred cleanup must leave later shard opens available");
|
||||
|
||||
disk1
|
||||
.release_snapshot_lease(bucket, &data_dir_path, lease)
|
||||
.await
|
||||
.expect("final lease release should reclaim the old data directory");
|
||||
assert!(matches!(disk1.read_all(bucket, &shard_path).await, Err(DiskError::FileNotFound)));
|
||||
}
|
||||
|
||||
/// Isolation guard: an armed barrier / observed object only affects its own
|
||||
/// object. A fan-out for a different (unobserved, unarmed) object must not be
|
||||
/// paused and must not accrue any tracked task count — so concurrent tests
|
||||
|
||||
@@ -555,7 +555,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
|
||||
fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
Self::update_file_info_quorum_hash(&mut hasher, meta);
|
||||
let digest = hasher.finalize();
|
||||
@@ -578,13 +578,6 @@ impl SetDisks {
|
||||
Self::update_hash_str(hasher, &meta.transition_tier);
|
||||
Self::update_hash_str(hasher, &meta.transitioned_objname);
|
||||
Self::update_hash_optional_uuid(hasher, meta.transition_version_id);
|
||||
Self::update_hash_optional_str(hasher, meta.transition_version.as_deref());
|
||||
hasher.update([match meta.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => 0,
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled => 1,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull => 2,
|
||||
rustfs_filemeta::TransitionVersionState::Exact => 3,
|
||||
}]);
|
||||
Self::update_hash_optional_u32(hasher, meta.mode);
|
||||
Self::update_hash_optional_u64(hasher, meta.written_by_version);
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -92,69 +92,6 @@ struct PartFailureSummary {
|
||||
bitrot_failure: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RecoverableMetaCandidate {
|
||||
identity: [u8; 32],
|
||||
file_info: FileInfo,
|
||||
data_count: usize,
|
||||
local_payload: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum DanglingDeleteSafety {
|
||||
UnsafeToDelete,
|
||||
NoRecoverableCandidate,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DanglingCheckPartsFailure {
|
||||
key: DanglingCheckPartsFailureKey,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type DanglingCheckPartsFailureKey = (String, String, usize);
|
||||
|
||||
#[cfg(test)]
|
||||
type DanglingCheckPartsFailures = HashMap<DanglingCheckPartsFailureKey, DiskError>;
|
||||
|
||||
#[cfg(test)]
|
||||
fn dangling_check_parts_failures() -> &'static std::sync::Mutex<DanglingCheckPartsFailures> {
|
||||
static FAILURES: std::sync::OnceLock<std::sync::Mutex<DanglingCheckPartsFailures>> = std::sync::OnceLock::new();
|
||||
FAILURES.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl DanglingCheckPartsFailure {
|
||||
fn install(bucket: &str, object: &str, disk_index: usize, error: DiskError) -> Self {
|
||||
let key = (bucket.to_string(), object.to_string(), disk_index);
|
||||
let previous = dangling_check_parts_failures()
|
||||
.lock()
|
||||
.expect("dangling check-parts failure registry should not poison")
|
||||
.insert(key.clone(), error);
|
||||
assert!(previous.is_none(), "dangling check-parts failure already installed");
|
||||
Self { key }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for DanglingCheckPartsFailure {
|
||||
fn drop(&mut self) {
|
||||
dangling_check_parts_failures()
|
||||
.lock()
|
||||
.expect("dangling check-parts failure registry should not poison")
|
||||
.remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn injected_dangling_check_parts_error(bucket: &str, object: &str, disk_index: usize) -> Option<DiskError> {
|
||||
dangling_check_parts_failures()
|
||||
.lock()
|
||||
.expect("dangling check-parts failure registry should not poison")
|
||||
.get(&(bucket.to_string(), object.to_string(), disk_index))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn first_unhealthy_part_summary(
|
||||
data_errs_by_part: &HashMap<usize, Vec<usize>>,
|
||||
parts: &[ObjectPartInfo],
|
||||
@@ -188,17 +125,39 @@ impl SetDisks {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, true)).await
|
||||
// `allow_meta_regen` is true on the first pass: a version whose data shards
|
||||
// physically survive (>= data_blocks) but whose xl.meta fell below
|
||||
// read-quorum is RESCUED (missing xl.meta regenerated) rather than
|
||||
// dangling-deleted. The re-drive after a rescue sets it false so the
|
||||
// regeneration can happen at most once (no unbounded recursion).
|
||||
Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, true)).await
|
||||
}
|
||||
|
||||
/// Best-effort orphan-data-dir reclaim for an object that is healthy on this
|
||||
/// set. Wraps [`Self::reclaim_orphan_data_dirs`] with the shared logging so
|
||||
/// both `heal_object` exits — the already-healthy early return and the
|
||||
/// post-heal tail — reclaim identically. Never fails the heal: delete errors
|
||||
/// are logged and swallowed. Callers must gate this on `!opts.dry_run`.
|
||||
async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
|
||||
match self.reclaim_orphan_data_dirs(bucket, object).await {
|
||||
Ok(removed) if removed > 0 => {
|
||||
info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
async fn heal_object_with_explicit_version_regen(
|
||||
async fn heal_object_with_regen(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_explicit_version_regen: bool,
|
||||
allow_meta_regen: bool,
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
info!(?opts, "Starting heal_object");
|
||||
|
||||
@@ -395,6 +354,22 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
// DATA-SAFETY GUARD (backlog#920, decision 1): before any
|
||||
// dangling delete, if the version's DATA shards physically
|
||||
// survive on >= data_blocks disks it is RECONSTRUCTABLE.
|
||||
// Regenerate the missing xl.meta from a surviving valid
|
||||
// FileInfo and re-drive the heal instead of destroying a
|
||||
// recoverable version. Torn writes (< data_blocks data
|
||||
// shards) fall through to the existing dangling behavior.
|
||||
if cannot_heal
|
||||
&& allow_meta_regen
|
||||
&& self
|
||||
.try_regenerate_recoverable_meta(bucket, object, &parts_metadata, &errs, &disks)
|
||||
.await?
|
||||
{
|
||||
return Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, false)).await;
|
||||
}
|
||||
|
||||
if cannot_heal {
|
||||
let total_disks = parts_metadata.len();
|
||||
let healthy_count = total_disks.saturating_sub(disks_to_heal_count);
|
||||
@@ -447,20 +422,6 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
// `disks_with_all_parts` normalizes conflicting entries
|
||||
// in `parts_metadata` to defaults. Re-read only before
|
||||
// destructive cleanup so the guard sees every original
|
||||
// identity.
|
||||
let (delete_guard_metadata, delete_guard_errs) =
|
||||
Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?;
|
||||
if self
|
||||
.dangling_delete_safety(bucket, object, &delete_guard_metadata, &delete_guard_errs, &disks)
|
||||
.await?
|
||||
== DanglingDeleteSafety::UnsafeToDelete
|
||||
{
|
||||
return Ok((result, Some(cannot_heal_err)));
|
||||
}
|
||||
|
||||
// Allow for dangling deletes, on versions that have DataDir missing etc.
|
||||
// this would end up restoring the correct readable versions.
|
||||
return match self
|
||||
@@ -876,25 +837,17 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if allow_explicit_version_regen
|
||||
&& !version_id.is_empty()
|
||||
// DATA-SAFETY GUARD (backlog#920, decision 1): meta quorum failed,
|
||||
// but the version's DATA may still physically survive on enough
|
||||
// disks (xl.meta lost on > parity disks while part files remain).
|
||||
// Rescue it by regenerating the missing xl.meta and re-driving heal
|
||||
// instead of dangling-deleting a reconstructable version.
|
||||
if allow_meta_regen
|
||||
&& self
|
||||
.try_regenerate_explicit_version_meta(bucket, object, version_id, &parts_metadata, &errs, &disks)
|
||||
.try_regenerate_recoverable_meta(bucket, object, &parts_metadata, &errs, &disks)
|
||||
.await?
|
||||
{
|
||||
return Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, false)).await;
|
||||
}
|
||||
|
||||
if self
|
||||
.dangling_delete_safety(bucket, object, &parts_metadata, &errs, &disks)
|
||||
.await?
|
||||
== DanglingDeleteSafety::UnsafeToDelete
|
||||
{
|
||||
return Ok((
|
||||
self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
|
||||
.await,
|
||||
Some(err),
|
||||
));
|
||||
return Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, false)).await;
|
||||
}
|
||||
|
||||
let data_errs_by_part = HashMap::new();
|
||||
@@ -930,226 +883,129 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
async fn try_regenerate_explicit_version_meta(
|
||||
/// backlog#920 (decision 1): rescue a version that meta-quorum logic would
|
||||
/// otherwise dangling-DELETE, when its DATA is still reconstructable.
|
||||
///
|
||||
/// Returns `Ok(true)` if the version was rescued (missing xl.meta regenerated
|
||||
/// on at least one disk, so a re-driven heal can reconstruct it), `Ok(false)`
|
||||
/// to fall through to the existing dangling-delete behavior.
|
||||
///
|
||||
/// Recoverability is computed by physically probing part files across ALL
|
||||
/// disks in the set with `check_parts` — including disks whose xl.meta is
|
||||
/// absent (a lost xl.meta does not lose the sibling `part.*` data). If at
|
||||
/// least `data_blocks` disks hold every part of a surviving valid FileInfo,
|
||||
/// the object is EC-reconstructable, so we regenerate that FileInfo's xl.meta
|
||||
/// on every disk whose metadata is absent (via `write_metadata`, which merges
|
||||
/// into any existing xl.meta). Delete markers, remote/transitioned versions,
|
||||
/// and genuine torn writes (< `data_blocks` surviving data shards) are NOT
|
||||
/// rescued — they keep the current dangling-delete-after-grace behavior, so no
|
||||
/// regression on those paths.
|
||||
async fn try_regenerate_recoverable_meta(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
parts_metadata: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> disk::error::Result<bool> {
|
||||
let Ok(version_id) = Uuid::parse_str(version_id) else {
|
||||
// A surviving valid, non-deleted, non-remote data FileInfo to rebuild from.
|
||||
let Some(surviving) = parts_metadata
|
||||
.iter()
|
||||
.find(|fi| fi.has_valid_erasure_geometry() && !fi.deleted && !fi.is_remote())
|
||||
.cloned()
|
||||
else {
|
||||
return Ok(false);
|
||||
};
|
||||
let candidates = parts_metadata
|
||||
.iter()
|
||||
.zip(errs.iter())
|
||||
.filter_map(|(file_info, err)| {
|
||||
(err.is_none()
|
||||
&& file_info_is_valid_for_metadata(file_info)
|
||||
&& file_info.version_id == Some(version_id)
|
||||
&& file_info.has_valid_erasure_geometry()
|
||||
&& !file_info.deleted
|
||||
&& !file_info.is_remote()
|
||||
&& file_info.data_dir.is_some()
|
||||
&& !file_info.parts.is_empty()
|
||||
&& file_info.erasure.data_blocks > 0
|
||||
&& file_info
|
||||
.erasure
|
||||
.data_blocks
|
||||
.checked_add(file_info.erasure.parity_blocks)
|
||||
.is_some_and(|shards| shards == disks.len()))
|
||||
.then_some(file_info)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let Some(candidate) = candidates.first().copied() else {
|
||||
|
||||
// Without a data_dir + parts there is no data to prove recoverable.
|
||||
if surviving.data_dir.is_none() || surviving.parts.is_empty() {
|
||||
return Ok(false);
|
||||
};
|
||||
let identity = Self::file_info_quorum_hash(candidate);
|
||||
if candidates
|
||||
.iter()
|
||||
.any(|file_info| Self::file_info_quorum_hash(file_info) != identity)
|
||||
{
|
||||
}
|
||||
let data_blocks = surviving.erasure.data_blocks;
|
||||
if data_blocks == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Physically probe part presence on EVERY online disk using the surviving
|
||||
// FileInfo's data_dir/parts. `check_parts` stats `object/<data_dir>/part.N`
|
||||
// directly, so it counts disks that still hold the data even if their
|
||||
// xl.meta was deleted.
|
||||
let mut available = 0usize;
|
||||
for disk in disks {
|
||||
let Some(disk) = disk else {
|
||||
return Ok(false);
|
||||
};
|
||||
match disk.check_parts(bucket, object, candidate).await {
|
||||
Ok(response)
|
||||
if !response.results.is_empty() && response.results.iter().all(|result| *result == CHECK_PART_SUCCESS) =>
|
||||
{
|
||||
available += 1;
|
||||
}
|
||||
Ok(_)
|
||||
| Err(
|
||||
DiskError::FileNotFound
|
||||
| DiskError::FileVersionNotFound
|
||||
| DiskError::PathNotFound
|
||||
| DiskError::VolumeNotFound,
|
||||
) => {}
|
||||
Err(_) => return Ok(false),
|
||||
for disk in disks.iter().flatten() {
|
||||
if let Ok(resp) = disk.check_parts(bucket, object, &surviving).await
|
||||
&& !resp.results.is_empty()
|
||||
&& resp.results.iter().all(|r| *r == CHECK_PART_SUCCESS)
|
||||
{
|
||||
available += 1;
|
||||
}
|
||||
}
|
||||
if available < candidate.erasure.data_blocks {
|
||||
|
||||
// Torn write: fewer than data_blocks surviving data shards is genuinely
|
||||
// unrecoverable — preserve the current dangling behavior (no resurrection).
|
||||
if available < data_blocks {
|
||||
debug!(
|
||||
bucket,
|
||||
object,
|
||||
available,
|
||||
data_blocks,
|
||||
"heal_object: version not reconstructable (torn write), keeping dangling behavior"
|
||||
);
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Reconstructable: regenerate the surviving xl.meta on every disk whose
|
||||
// metadata is absent so the version regains read-quorum. Each disk gets its
|
||||
// OWN shard index: the disk at physical position `index` holds shard
|
||||
// `distribution[index]` (mirrors `shuffle_disks` + the write path's
|
||||
// `erasure.index = shuffled_pos + 1`). Copying the surviving disk's index
|
||||
// verbatim would write an inconsistent xl.meta that the re-heal then treats
|
||||
// as corrupt.
|
||||
let distribution = &surviving.erasure.distribution;
|
||||
let mut wrote = 0usize;
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let Some(disk) = disk else {
|
||||
return Ok(false);
|
||||
};
|
||||
let metadata_absent = matches!(
|
||||
let Some(disk) = disk else { continue };
|
||||
let meta_absent = matches!(
|
||||
errs.get(index).and_then(Option::as_ref),
|
||||
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
||||
);
|
||||
if !metadata_absent {
|
||||
) || !parts_metadata.get(index).map(FileInfo::is_valid).unwrap_or(false);
|
||||
if !meta_absent {
|
||||
continue;
|
||||
}
|
||||
let Some(&shard_index) = candidate.erasure.distribution.get(index) else {
|
||||
return Ok(false);
|
||||
// Without a known shard index for this position we cannot write a
|
||||
// consistent xl.meta; leave it for the normal heal to reconstruct.
|
||||
let Some(&shard_index) = distribution.get(index) else {
|
||||
continue;
|
||||
};
|
||||
let mut regenerated = candidate.clone();
|
||||
regenerated.fresh = false;
|
||||
regenerated.erasure.index = shard_index;
|
||||
match disk.write_metadata("", bucket, object, regenerated).await {
|
||||
let mut regen = surviving.clone();
|
||||
regen.fresh = false; // merge into any existing xl.meta on the disk
|
||||
regen.erasure.index = shard_index;
|
||||
match disk.write_metadata("", bucket, object, regen).await {
|
||||
Ok(()) => wrote += 1,
|
||||
Err(error) => {
|
||||
Err(e) => {
|
||||
warn!(
|
||||
bucket,
|
||||
object,
|
||||
disk_index = index,
|
||||
error = %error,
|
||||
"failed to regenerate recoverable xl.meta"
|
||||
error = %e,
|
||||
"heal_object: failed to regenerate recoverable xl.meta on disk"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(wrote > 0)
|
||||
}
|
||||
|
||||
/// Best-effort orphan-data-dir reclaim for an object that is healthy on this
|
||||
/// set. Wraps [`Self::reclaim_orphan_data_dirs`] with the shared logging so
|
||||
/// both `heal_object` exits — the already-healthy early return and the
|
||||
/// post-heal tail — reclaim identically. Never fails the heal: delete errors
|
||||
/// are logged and swallowed. Callers must gate this on `!opts.dry_run`.
|
||||
async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
|
||||
match self.reclaim_orphan_data_dirs(bucket, object).await {
|
||||
Ok(removed) if removed > 0 => {
|
||||
info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Prevent dangling cleanup when surviving state cannot prove that deletion
|
||||
/// is safe. Part presence proves only recoverability, never commit: the write
|
||||
/// path can durably rename data before xl.meta is committed.
|
||||
async fn dangling_delete_safety(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
parts_metadata: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> disk::error::Result<DanglingDeleteSafety> {
|
||||
if disks.iter().any(Option::is_none)
|
||||
|| errs.iter().flatten().any(|err| {
|
||||
!matches!(
|
||||
err,
|
||||
DiskError::FileNotFound
|
||||
| DiskError::FileVersionNotFound
|
||||
| DiskError::PathNotFound
|
||||
| DiskError::VolumeNotFound
|
||||
)
|
||||
})
|
||||
{
|
||||
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
||||
if wrote == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut candidates = Vec::<RecoverableMetaCandidate>::with_capacity(parts_metadata.len());
|
||||
for (fi, err) in parts_metadata.iter().zip(errs.iter()) {
|
||||
if err.is_some() || !file_info_is_valid_for_metadata(fi) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let identity = Self::file_info_quorum_hash(fi);
|
||||
if !candidates.iter().any(|candidate| candidate.identity == identity) {
|
||||
let local_payload = fi.has_valid_erasure_geometry()
|
||||
&& !fi.deleted
|
||||
&& !fi.is_remote()
|
||||
&& fi.data_dir.is_some()
|
||||
&& !fi.parts.is_empty()
|
||||
&& fi.erasure.data_blocks > 0
|
||||
&& fi
|
||||
.erasure
|
||||
.data_blocks
|
||||
.checked_add(fi.erasure.parity_blocks)
|
||||
.is_some_and(|shards| shards == disks.len());
|
||||
candidates.push(RecoverableMetaCandidate {
|
||||
identity,
|
||||
file_info: fi.clone(),
|
||||
data_count: 0,
|
||||
local_payload,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if candidates
|
||||
.iter()
|
||||
.any(|candidate| candidate.file_info.deleted || candidate.file_info.is_remote())
|
||||
|| candidates.len() > 1
|
||||
{
|
||||
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
||||
}
|
||||
|
||||
for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
|
||||
for (disk_index, disk) in disks.iter().enumerate() {
|
||||
let Some(disk) = disk else {
|
||||
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
||||
};
|
||||
#[cfg(test)]
|
||||
let check_result = match injected_dangling_check_parts_error(bucket, object, disk_index) {
|
||||
Some(error) => Err(error),
|
||||
None => disk.check_parts(bucket, object, &candidate.file_info).await,
|
||||
};
|
||||
#[cfg(not(test))]
|
||||
let check_result = disk.check_parts(bucket, object, &candidate.file_info).await;
|
||||
|
||||
match check_result {
|
||||
Ok(resp) if !resp.results.is_empty() && resp.results.iter().all(|result| *result == CHECK_PART_SUCCESS) => {
|
||||
candidate.data_count += 1;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(
|
||||
DiskError::FileNotFound
|
||||
| DiskError::FileVersionNotFound
|
||||
| DiskError::PathNotFound
|
||||
| DiskError::VolumeNotFound,
|
||||
) => {}
|
||||
Err(_) => return Ok(DanglingDeleteSafety::UnsafeToDelete),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(
|
||||
if candidates
|
||||
.iter()
|
||||
.any(|candidate| candidate.local_payload && candidate.data_count >= candidate.file_info.erasure.data_blocks)
|
||||
{
|
||||
DanglingDeleteSafety::UnsafeToDelete
|
||||
} else {
|
||||
DanglingDeleteSafety::NoRecoverableCandidate
|
||||
},
|
||||
)
|
||||
info!(
|
||||
bucket,
|
||||
object,
|
||||
available,
|
||||
data_blocks,
|
||||
regenerated_meta_disks = wrote,
|
||||
"heal_object: rescued reconstructable sub-quorum version by regenerating xl.meta"
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn heal_object_dir_locked(
|
||||
@@ -1537,7 +1393,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
|
||||
#[cfg(test)]
|
||||
mod heal_result_report_tests {
|
||||
use super::{DanglingCheckPartsFailure, DanglingDeleteSafety, SetDisks};
|
||||
use super::SetDisks;
|
||||
use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::error::DiskError;
|
||||
@@ -1549,12 +1405,10 @@ mod heal_result_report_tests {
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::{config::storageclass, store::init_format::save_format_file};
|
||||
use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
|
||||
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
|
||||
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo};
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::RwLock;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn real_disk() -> (TempDir, Endpoint, DiskStore) {
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
@@ -1592,68 +1446,6 @@ mod heal_result_report_tests {
|
||||
.await
|
||||
}
|
||||
|
||||
fn meta_regen_test_fileinfo(object: &str, data_dir: Uuid, mod_time: i64, disk_index: usize) -> FileInfo {
|
||||
let mut fi = FileInfo::new(object, 2, 2);
|
||||
fi.data_dir = Some(data_dir);
|
||||
fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(mod_time).expect("test timestamp should parse"));
|
||||
fi.size = 1;
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.erasure.index = fi.erasure.distribution[disk_index];
|
||||
fi
|
||||
}
|
||||
|
||||
async fn meta_regen_test_set(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data_dirs: &[(Uuid, usize)],
|
||||
) -> (Vec<TempDir>, Arc<SetDisks>, Vec<Option<DiskStore>>) {
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let (temp_dir, endpoint, disk) = real_disk().await;
|
||||
disk.make_volume(bucket).await.expect("test bucket should be created");
|
||||
for (data_dir, shard_count) in data_dirs {
|
||||
if disk_index >= *shard_count {
|
||||
continue;
|
||||
}
|
||||
let part_dir = temp_dir.path().join(bucket).join(object).join(data_dir.to_string());
|
||||
tokio::fs::create_dir_all(&part_dir)
|
||||
.await
|
||||
.expect("test data directory should be created");
|
||||
tokio::fs::write(part_dir.join("part.1"), [1u8; 2])
|
||||
.await
|
||||
.expect("test data shard should be written");
|
||||
}
|
||||
temp_dirs.push(temp_dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
let set = set_disks_with(disks.clone(), endpoints, 2).await;
|
||||
(temp_dirs, set, disks)
|
||||
}
|
||||
|
||||
async fn seed_meta_regen_test_metadata(
|
||||
disks: &[Option<DiskStore>],
|
||||
disk_index: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
file_info: &FileInfo,
|
||||
) {
|
||||
disks[disk_index]
|
||||
.as_ref()
|
||||
.expect("metadata test disk should be online")
|
||||
.write_metadata("", bucket, object, file_info.clone())
|
||||
.await
|
||||
.expect("test metadata should be written");
|
||||
}
|
||||
|
||||
async fn formatted_single_disk_no_parity_set() -> (TempDir, Arc<SetDisks>) {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
@@ -1961,312 +1753,6 @@ mod heal_result_report_tests {
|
||||
assert_eq!(result.before.drives[3].state, DriveState::Ok.to_string());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_delete_guard_preserves_conflicting_identities_without_writing_metadata() {
|
||||
let bucket = "bucket-delete-guard-conflict";
|
||||
let object = "object.bin";
|
||||
let old_data_dir = Uuid::parse_str("99999999-9999-9999-9999-999999999999").expect("old data dir should parse");
|
||||
let new_data_dir = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa").expect("new data dir should parse");
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(old_data_dir, 4), (new_data_dir, 2)]).await;
|
||||
let version_id = Uuid::parse_str("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb").expect("version id should parse");
|
||||
let mut metadata = vec![
|
||||
meta_regen_test_fileinfo(object, old_data_dir, 9, 0),
|
||||
meta_regen_test_fileinfo(object, new_data_dir, 10, 1),
|
||||
FileInfo::default(),
|
||||
FileInfo::default(),
|
||||
];
|
||||
metadata[0].version_id = Some(version_id);
|
||||
metadata[1].version_id = Some(version_id);
|
||||
assert_eq!(
|
||||
metadata[0].version_id, metadata[1].version_id,
|
||||
"the conflicting candidates must share one version id"
|
||||
);
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata[0]).await;
|
||||
seed_meta_regen_test_metadata(&disks, 1, bucket, object, &metadata[1]).await;
|
||||
let errs = vec![None, None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)];
|
||||
|
||||
assert!(
|
||||
set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
|
||||
.await
|
||||
.expect("conflicting identities should be classified")
|
||||
== DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
let reversed = vec![
|
||||
metadata[1].clone(),
|
||||
metadata[0].clone(),
|
||||
FileInfo::default(),
|
||||
FileInfo::default(),
|
||||
];
|
||||
assert!(
|
||||
set.dangling_delete_safety(bucket, object, &reversed, &errs, &disks)
|
||||
.await
|
||||
.expect("reversed identities should be classified")
|
||||
== DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
let version_id = version_id.to_string();
|
||||
assert!(
|
||||
!set.try_regenerate_explicit_version_meta(bucket, object, &version_id, &metadata, &errs, &disks)
|
||||
.await
|
||||
.expect("conflicting explicit-version candidates should be rejected"),
|
||||
"an explicit version must not select between conflicting metadata identities"
|
||||
);
|
||||
for disk_index in [2, 3] {
|
||||
assert!(
|
||||
matches!(
|
||||
disks[disk_index]
|
||||
.as_ref()
|
||||
.expect("test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"the delete guard must not manufacture metadata on missing disks"
|
||||
);
|
||||
}
|
||||
let old = disks[0]
|
||||
.as_ref()
|
||||
.expect("first test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("old metadata should remain readable");
|
||||
let new = disks[1]
|
||||
.as_ref()
|
||||
.expect("second test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("new metadata should remain readable");
|
||||
assert_eq!(old.data_dir, Some(old_data_dir));
|
||||
assert_eq!(new.data_dir, Some(new_data_dir));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_meta_quorum_failure_preserves_reconstructable_uncommitted_candidate() {
|
||||
let bucket = "bucket-delete-guard-reconstructable";
|
||||
let object = "object.bin";
|
||||
let data_dir = Uuid::parse_str("33333333-3333-3333-3333-333333333333").expect("data dir should parse");
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
|
||||
let metadata = [
|
||||
meta_regen_test_fileinfo(object, data_dir, 3, 0),
|
||||
FileInfo::default(),
|
||||
FileInfo::default(),
|
||||
FileInfo::default(),
|
||||
];
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata[0]).await;
|
||||
let (observed_metadata, observed_errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", true, true, false)
|
||||
.await
|
||||
.expect("test metadata should be readable across the set");
|
||||
assert_eq!(
|
||||
set.dangling_delete_safety(bucket, object, &observed_metadata, &observed_errs, &disks)
|
||||
.await
|
||||
.expect("observed reconstructable candidate should be classified"),
|
||||
DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
|
||||
let (_, err) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("unsafe dangling state should be reported without deletion");
|
||||
assert_eq!(err, Some(DiskError::FileNotFound));
|
||||
let surviving = disks[0]
|
||||
.as_ref()
|
||||
.expect("first test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("the only metadata copy must be preserved");
|
||||
assert_eq!(surviving.data_dir, Some(data_dir));
|
||||
assert!(
|
||||
matches!(
|
||||
disks[1]
|
||||
.as_ref()
|
||||
.expect("second test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"the delete guard must not propagate metadata"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_meta_quorum_failure_preserves_candidate_when_required_shard_disk_is_offline() {
|
||||
let bucket = "bucket-delete-guard-offline";
|
||||
let object = "object.bin";
|
||||
let data_dir = Uuid::parse_str("44444444-4444-4444-4444-444444444444").expect("data dir should parse");
|
||||
let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
|
||||
let metadata = meta_regen_test_fileinfo(object, data_dir, 4, 0);
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
|
||||
set.disks.write().await[1] = None;
|
||||
|
||||
let (_, err) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("offline shard state should be reported without deletion");
|
||||
assert_eq!(err, Some(DiskError::FileNotFound));
|
||||
let surviving = disks[0]
|
||||
.as_ref()
|
||||
.expect("first test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("offline uncertainty must preserve the surviving metadata");
|
||||
assert_eq!(surviving.data_dir, Some(data_dir));
|
||||
assert!(
|
||||
temp_dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1")
|
||||
.is_file(),
|
||||
"offline uncertainty must preserve the last online shard"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_meta_quorum_failure_preserves_candidate_when_part_probe_times_out() {
|
||||
let bucket = "bucket-delete-guard-timeout";
|
||||
let object = "object.bin";
|
||||
let data_dir = Uuid::parse_str("55555555-5555-5555-5555-555555555555").expect("data dir should parse");
|
||||
let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
|
||||
let metadata = meta_regen_test_fileinfo(object, data_dir, 5, 0);
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
|
||||
let _failure = DanglingCheckPartsFailure::install(bucket, object, 1, DiskError::Timeout);
|
||||
|
||||
let (_, err) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("part probe timeout should be reported without deletion");
|
||||
assert_eq!(err, Some(DiskError::FileNotFound));
|
||||
let surviving = disks[0]
|
||||
.as_ref()
|
||||
.expect("first test disk should be online")
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("probe uncertainty must preserve the surviving metadata");
|
||||
assert_eq!(surviving.data_dir, Some(data_dir));
|
||||
assert!(
|
||||
temp_dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1")
|
||||
.is_file(),
|
||||
"probe uncertainty must preserve the last confirmed shard"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_delete_guard_ignores_set_incompatible_geometry() {
|
||||
let bucket = "bucket-delete-guard-short-geometry";
|
||||
let object = "object.bin";
|
||||
let data_dir = Uuid::parse_str("abababab-abab-abab-abab-abababababab").expect("data dir should parse");
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 1)]).await;
|
||||
let mut candidate = FileInfo::new(object, 1, 0);
|
||||
candidate.data_dir = Some(data_dir);
|
||||
candidate.mod_time = Some(OffsetDateTime::from_unix_timestamp(18).expect("timestamp should parse"));
|
||||
candidate.size = 1;
|
||||
candidate.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
candidate.erasure.index = candidate.erasure.distribution[0];
|
||||
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &candidate).await;
|
||||
let metadata = vec![candidate, FileInfo::default(), FileInfo::default(), FileInfo::default()];
|
||||
let errs = vec![
|
||||
None,
|
||||
Some(DiskError::FileNotFound),
|
||||
Some(DiskError::FileNotFound),
|
||||
Some(DiskError::FileNotFound),
|
||||
];
|
||||
|
||||
assert!(
|
||||
set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
|
||||
.await
|
||||
.expect("set-incompatible geometry should be classified")
|
||||
== DanglingDeleteSafety::NoRecoverableCandidate
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_delete_guard_preserves_delete_marker_and_remote_metadata() {
|
||||
let bucket = "bucket-delete-guard-nonlocal";
|
||||
let object = "object.bin";
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
|
||||
let marker = FileInfo {
|
||||
name: object.to_string(),
|
||||
version_id: Some(Uuid::parse_str("eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee").expect("version id should parse")),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(14).expect("marker timestamp should parse")),
|
||||
..Default::default()
|
||||
};
|
||||
let remote_dir = Uuid::parse_str("89898989-8989-8989-8989-898989898989").expect("remote data dir should parse");
|
||||
let mut remote = meta_regen_test_fileinfo(object, remote_dir, 15, 1);
|
||||
remote.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
remote.transition_tier = "WARM".to_string();
|
||||
remote.transitioned_objname = "remote/object.bin".to_string();
|
||||
|
||||
for metadata in [marker, remote] {
|
||||
let candidates = vec![metadata, FileInfo::default(), FileInfo::default(), FileInfo::default()];
|
||||
let errs = vec![
|
||||
None,
|
||||
Some(DiskError::FileNotFound),
|
||||
Some(DiskError::FileNotFound),
|
||||
Some(DiskError::FileNotFound),
|
||||
];
|
||||
assert_eq!(
|
||||
set.dangling_delete_safety(bucket, object, &candidates, &errs, &disks)
|
||||
.await
|
||||
.expect("non-local metadata should be classified"),
|
||||
DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_delete_guard_preserves_metadata_read_uncertainty() {
|
||||
let bucket = "bucket-delete-guard-read-error";
|
||||
let object = "object.bin";
|
||||
let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
|
||||
let metadata = vec![FileInfo::default(); disks.len()];
|
||||
|
||||
for read_error in [DiskError::Timeout, DiskError::DiskAccessDenied, DiskError::DiskNotFound] {
|
||||
let mut errs = vec![Some(DiskError::FileNotFound); disks.len()];
|
||||
errs[0] = Some(read_error);
|
||||
assert_eq!(
|
||||
set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
|
||||
.await
|
||||
.expect("metadata read uncertainty should be classified"),
|
||||
DanglingDeleteSafety::UnsafeToDelete
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_no_parity_bitrot_reports_unrecoverable_integrity_failure() {
|
||||
let (dir, set) = formatted_single_disk_no_parity_set().await;
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::multipart_listing::paginate_multipart_listing;
|
||||
use futures::{StreamExt, stream};
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
@@ -36,7 +36,6 @@ const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
|
||||
#[cfg(test)]
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum MultipartCommitPause {
|
||||
PutPartBeforeLockAcquire,
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterRename,
|
||||
BeforeLockLost,
|
||||
@@ -48,10 +47,9 @@ struct MultipartCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
pause: MultipartCommitPause,
|
||||
expected_arrivals: usize,
|
||||
arrivals: AtomicUsize,
|
||||
armed: AtomicBool,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Semaphore,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -66,24 +64,13 @@ static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc
|
||||
#[cfg(test)]
|
||||
impl MultipartCommitBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
|
||||
Self::install_for_arrivals(bucket, object, pause, 1)
|
||||
}
|
||||
|
||||
pub(crate) fn install_for_arrivals(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
pause: MultipartCommitPause,
|
||||
expected_arrivals: usize,
|
||||
) -> Self {
|
||||
assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival");
|
||||
let state = Arc::new(MultipartCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
pause,
|
||||
expected_arrivals,
|
||||
arrivals: AtomicUsize::new(0),
|
||||
armed: AtomicBool::new(true),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Semaphore::new(0),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = MULTIPART_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -96,28 +83,20 @@ impl MultipartCommitBarrier {
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let arrived = self.state.arrived.notified();
|
||||
if self.state.arrivals.load(Ordering::Acquire) >= self.state.expected_arrivals {
|
||||
return;
|
||||
}
|
||||
arrived.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("multipart completion should reach the deterministic commit barrier");
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("multipart completion should reach the deterministic commit barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.add_permits(self.state.expected_arrivals);
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for MultipartCommitBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
self.state.release.notify_one();
|
||||
let mut slot = MULTIPART_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -138,20 +117,10 @@ async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartComm
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier
|
||||
&& let Ok(previous) = barrier.arrivals.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
|
||||
(current < barrier.expected_arrivals).then_some(current + 1)
|
||||
})
|
||||
&& barrier.armed.swap(false, Ordering::AcqRel)
|
||||
{
|
||||
let arrival = previous + 1;
|
||||
if arrival == barrier.expected_arrivals {
|
||||
barrier.arrived.notify_one();
|
||||
}
|
||||
barrier
|
||||
.release
|
||||
.acquire()
|
||||
.await
|
||||
.expect("multipart commit barrier should remain open")
|
||||
.forget();
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -724,8 +693,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockAcquire).await;
|
||||
// Serialize only the commit (rename_part), not the whole upload. Each
|
||||
// concurrent stream writes to its own unique temp dir (see `tmp_part`
|
||||
// above), so the encode/stream phase never conflicts and must stay
|
||||
|
||||
@@ -25,10 +25,7 @@ use crate::set_disk::read::GetObjectDownstreamWriter;
|
||||
|
||||
use crate::bucket::lifecycle::{
|
||||
tier_delete_journal::{persist_tier_delete_journal_entry, remove_tier_delete_journal_entry},
|
||||
tier_sweeper::{
|
||||
Jentry, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_object_from_remote_tier_with_lease_idempotent,
|
||||
},
|
||||
tier_sweeper::{Jentry, RemoteTierDeleteOutcome, delete_object_from_remote_tier_with_lease_idempotent},
|
||||
transition_transaction::{
|
||||
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
|
||||
TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record,
|
||||
@@ -550,7 +547,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
&self.ctx.tier_config_mgr(),
|
||||
)
|
||||
.await?;
|
||||
return Ok(finish_set_disk_read_lock(gr, read_lock_guard.take(), None, bucket, object));
|
||||
return Ok(finish_set_disk_read_lock(
|
||||
gr,
|
||||
read_lock_guard.take(),
|
||||
lock_optimization_enabled,
|
||||
bucket,
|
||||
object,
|
||||
));
|
||||
}
|
||||
|
||||
// App-layer object data cache probe: metadata (etag/size) is resolved
|
||||
@@ -677,18 +680,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
return Ok(reader);
|
||||
}
|
||||
|
||||
let snapshot_lease = if lock_optimization_enabled {
|
||||
match fi.data_dir.filter(|data_dir| !data_dir.is_nil()) {
|
||||
Some(data_dir) => {
|
||||
let data_dir_path = format!("{object}/{data_dir}");
|
||||
acquire_snapshot_leases(&disks, bucket, &data_dir_path, fi.erasure.data_blocks).await
|
||||
}
|
||||
None => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
match codec_streaming_gate.decision {
|
||||
GetCodecStreamingDecision::Use => {
|
||||
match Self::get_object_decode_reader_with_fileinfo(
|
||||
@@ -717,7 +708,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
// Carry the hook probe result so the app layer skips its
|
||||
// now-redundant lookup on the streaming miss path (ODC-16).
|
||||
reader.body_source = body_source;
|
||||
return Ok(finish_set_disk_read_lock(reader, read_lock_guard.take(), snapshot_lease, bucket, object));
|
||||
return Ok(finish_set_disk_read_lock(
|
||||
reader,
|
||||
read_lock_guard.take(),
|
||||
lock_optimization_enabled,
|
||||
bucket,
|
||||
object,
|
||||
));
|
||||
}
|
||||
core::io_primitives::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
|
||||
record_get_codec_streaming_gate_decision(
|
||||
@@ -756,22 +753,15 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
let set_index = self.set_index;
|
||||
let pool_index = self.pool_index;
|
||||
let skip_verify = opts.skip_verify_bitrot;
|
||||
let producer_snapshot_lease = snapshot_lease.clone();
|
||||
if let Some(lease) = snapshot_lease {
|
||||
if lock_optimization_enabled {
|
||||
release_materialized_read_lock(&bucket, &object, read_lock_guard.take());
|
||||
reader.stream = Box::new(SnapshotLeaseReader {
|
||||
inner: reader.stream,
|
||||
lease: Some(lease),
|
||||
terminal_error: false,
|
||||
});
|
||||
debug!(bucket, object, "Lock optimization: replaced read lock with snapshot leases");
|
||||
debug!(bucket, object, "Lock optimization: released read lock before streaming read");
|
||||
}
|
||||
|
||||
// The producer shares the lease lifetime with the body so cancellation
|
||||
// cannot release the snapshot while the duplex task is still unwinding.
|
||||
// When lock optimization is disabled, keep the read-lock guard in the
|
||||
// task so it lives for the duration of the streaming read.
|
||||
tokio::spawn(async move {
|
||||
let _guard = read_lock_guard;
|
||||
let _snapshot_lease = producer_snapshot_lease;
|
||||
let mut writer = GetObjectDownstreamWriter::new(wd);
|
||||
// Do not wrap the entire read+write pipeline in `disk_read_timeout`.
|
||||
// `get_object_with_fileinfo` also waits on `writer`, so an outer timeout
|
||||
@@ -1673,11 +1663,7 @@ pub(crate) async fn cleanup_uncommitted_transition_upload(
|
||||
cleanup_version: &str,
|
||||
version_id_exact: bool,
|
||||
) -> std::io::Result<RemoteTierDeleteOutcome> {
|
||||
if version_id_exact {
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(object, cleanup_version, lease).await
|
||||
} else {
|
||||
delete_object_from_remote_tier_with_lease_idempotent(object, cleanup_version, lease, false).await
|
||||
}
|
||||
delete_object_from_remote_tier_with_lease_idempotent(object, cleanup_version, lease, version_id_exact).await
|
||||
}
|
||||
|
||||
fn log_transition_upload_cleanup_failure(lease: &TierOperationLease, object: &str, cleanup_version: &str, err: &std::io::Error) {
|
||||
@@ -1814,7 +1800,7 @@ impl Drop for TransitionUploadCleanup {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_rejected_transition_upload_durably(
|
||||
async fn cleanup_rejected_transition_upload_durably(
|
||||
lease: &TierOperationLease,
|
||||
object: &str,
|
||||
cleanup_version: &str,
|
||||
@@ -1827,13 +1813,6 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably(
|
||||
tier_name: lease.tier_name().to_string(),
|
||||
backend_identity: Some(lease.backend_identity()),
|
||||
version_id_exact,
|
||||
version_state: if !version_id_exact {
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled
|
||||
} else if cleanup_version == "null" {
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull
|
||||
} else {
|
||||
rustfs_filemeta::TransitionVersionState::Exact
|
||||
},
|
||||
};
|
||||
|
||||
let journal_error = if let Some(api) = api.as_ref() {
|
||||
@@ -1993,83 +1972,12 @@ async fn advance_and_save_transition_transaction(
|
||||
next: TransitionTransactionState,
|
||||
remote_version: Option<TransitionRemoteVersion>,
|
||||
) -> Result<()> {
|
||||
#[cfg(test)]
|
||||
record_transition_uploaded_save_attempt(transaction, next);
|
||||
transaction
|
||||
.advance(transaction.fence(), next, remote_version)
|
||||
.map_err(Error::other)?;
|
||||
save_transition_transaction_if_available(api, transaction).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct TransitionUploadedSaveProbeState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
attempts: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct TransitionUploadedSaveProbe {
|
||||
state: Arc<TransitionUploadedSaveProbeState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static TRANSITION_UPLOADED_SAVE_PROBE: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionUploadedSaveProbeState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl TransitionUploadedSaveProbe {
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(TransitionUploadedSaveProbeState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
attempts: std::sync::atomic::AtomicUsize::new(0),
|
||||
});
|
||||
let mut slot = TRANSITION_UPLOADED_SAVE_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition uploaded-save probe mutex should not poison");
|
||||
assert!(slot.is_none(), "transition uploaded-save probe must be installed by one test at a time");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
drop(slot);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
fn attempts(&self) -> usize {
|
||||
self.state.attempts.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for TransitionUploadedSaveProbe {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = TRANSITION_UPLOADED_SAVE_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition uploaded-save probe mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction, next: TransitionTransactionState) {
|
||||
if next != TransitionTransactionState::Uploaded {
|
||||
return;
|
||||
}
|
||||
let state = TRANSITION_UPLOADED_SAVE_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition uploaded-save probe mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|state| state.bucket == transaction.source.bucket && state.object == transaction.source.object)
|
||||
.cloned();
|
||||
if let Some(state) = state {
|
||||
state.attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_transition_transaction_if_available(api: Option<&Arc<ECStore>>, transaction_id: Uuid) -> Result<()> {
|
||||
if let Some(api) = api {
|
||||
return delete_transition_transaction_record(api.clone(), transaction_id).await;
|
||||
@@ -2320,52 +2228,11 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
|
||||
}
|
||||
}
|
||||
|
||||
fn persisted_transition_version(
|
||||
remote_version: &str,
|
||||
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
|
||||
persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_enabled() -> bool {
|
||||
remote_version_state_writer_enabled_for(
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
),
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
|
||||
requested && fleet_confirmed
|
||||
}
|
||||
|
||||
fn persisted_transition_version_with_gate(
|
||||
remote_version: &str,
|
||||
remote_version_state_writer_enabled: bool,
|
||||
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
|
||||
fn parse_transition_version_id(remote_version: &str) -> std::result::Result<Option<Uuid>, uuid::Error> {
|
||||
if remote_version.is_empty() {
|
||||
return Ok((None, rustfs_filemeta::TransitionVersionState::KnownDisabled));
|
||||
}
|
||||
|
||||
match Uuid::parse_str(remote_version) {
|
||||
Ok(version_id) if version_id.is_nil() => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier returned a nil object version ID",
|
||||
)),
|
||||
Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
|
||||
Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"opaque remote tier versions require the operator-attested fleet gate",
|
||||
)),
|
||||
Err(_) if remote_version == "null" => {
|
||||
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
|
||||
}
|
||||
Err(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
|
||||
return Ok(None);
|
||||
}
|
||||
Uuid::parse_str(remote_version).map(|version_id| (!version_id.is_nil()).then_some(version_id))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2603,20 +2470,16 @@ mod transition_upload_completion_tests {
|
||||
|
||||
#[cfg(test)]
|
||||
mod transition_version_id_tests {
|
||||
use super::{
|
||||
TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
|
||||
remote_version_state_writer_enabled_for,
|
||||
};
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use super::{TransitionUploadCandidate, parse_transition_version_id};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn normalizes_persisted_unversioned_ids_and_preserves_put_constraints() {
|
||||
assert_eq!(parse_transition_version_id("").expect("empty remote version should be valid"), None);
|
||||
assert_eq!(
|
||||
persisted_transition_version("").expect("empty remote version identifies an unversioned tier"),
|
||||
(None, TransitionVersionState::KnownDisabled)
|
||||
parse_transition_version_id(&Uuid::nil().to_string()).expect("nil remote version should be valid"),
|
||||
None
|
||||
);
|
||||
assert!(persisted_transition_version(&Uuid::nil().to_string()).is_err());
|
||||
let nil_put_response = Uuid::nil().to_string();
|
||||
let nil_candidate = TransitionUploadCandidate::from_put_response(nil_put_response.clone());
|
||||
assert_eq!(nil_candidate.cleanup_version(), nil_put_response);
|
||||
@@ -2628,14 +2491,12 @@ mod transition_version_id_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_uuid_and_gates_opaque_remote_ids() {
|
||||
fn preserves_valid_remote_id_and_rejects_invalid_text() {
|
||||
let version_id = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
persisted_transition_version(&version_id.to_string()).expect("UUID remote version"),
|
||||
(Some(version_id.to_string()), TransitionVersionState::Exact)
|
||||
parse_transition_version_id(&version_id.to_string()).expect("UUID remote version should be valid"),
|
||||
Some(version_id)
|
||||
);
|
||||
assert!(persisted_transition_version("null").is_err());
|
||||
assert!(persisted_transition_version("opaque-version-token").is_err());
|
||||
assert_eq!(
|
||||
TransitionUploadCandidate::from_put_response(version_id.to_string()).cleanup_version(),
|
||||
version_id.to_string()
|
||||
@@ -2644,44 +2505,7 @@ mod transition_version_id_tests {
|
||||
TransitionUploadCandidate::from_put_response("opaque-version-token".to_string()).cleanup_version(),
|
||||
"opaque-version-token"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
|
||||
for (case, requested, fleet_confirmed, expected) in [
|
||||
("old defaults", false, false, false),
|
||||
("missing fleet confirmation", true, false, false),
|
||||
("missing local opt-in", false, true, false),
|
||||
("explicitly unconfirmed fleet", true, false, false),
|
||||
("rolled-back writer", false, true, false),
|
||||
("fully upgraded fleet", true, true, true),
|
||||
] {
|
||||
assert_eq!(remote_version_state_writer_enabled_for(requested, fleet_confirmed), expected, "{case}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fleet_gate_enables_null_and_opaque_remote_version_states() {
|
||||
for (remote_version, expected) in [
|
||||
("null", (Some("null".to_string()), TransitionVersionState::SuspendedNull)),
|
||||
(
|
||||
"opaque-version-token",
|
||||
(Some("opaque-version-token".to_string()), TransitionVersionState::Exact),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
persisted_transition_version_with_gate(remote_version, false).is_err(),
|
||||
"missing fleet confirmation must reject {remote_version:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
persisted_transition_version_with_gate(remote_version, true).expect("fleet-confirmed state must be persisted"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
persisted_transition_version_with_gate("", true).expect("empty remote version identifies an unversioned tier"),
|
||||
(None, TransitionVersionState::KnownDisabled)
|
||||
);
|
||||
assert!(parse_transition_version_id("not-a-uuid").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3951,20 +3775,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
|
||||
return Err(err.into());
|
||||
}
|
||||
let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) {
|
||||
Ok(version) => version,
|
||||
Err(err) => {
|
||||
let cleanup_api = transition_cleanup_store(&self.ctx).await;
|
||||
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
|
||||
return Err(StorageError::Io(std::io::Error::other(format!(
|
||||
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
|
||||
))));
|
||||
}
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
if let Err(err) = advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
@@ -3982,6 +3792,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
|
||||
return Err(err);
|
||||
}
|
||||
let transition_version_id = match parse_transition_version_id(candidate.remote_version()) {
|
||||
Ok(version_id) => version_id,
|
||||
Err(err) => {
|
||||
if upload_cleanup.cleanup().await.is_ok() {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
}
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
|
||||
let mut commit_opts = opts.clone();
|
||||
commit_opts.no_lock = true;
|
||||
@@ -4039,11 +3859,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
current_fi.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
current_fi.transitioned_objname = dest_obj;
|
||||
current_fi.transition_tier = opts.transition.tier.clone();
|
||||
current_fi.transition_version_id = transition_version_id
|
||||
.as_deref()
|
||||
.and_then(|version_id| Uuid::parse_str(version_id).ok());
|
||||
current_fi.transition_version = transition_version_id;
|
||||
current_fi.transition_version_state = transition_version_state;
|
||||
current_fi.transition_version_id = transition_version_id;
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut current_fi.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
@@ -4260,7 +4076,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let mut p_reader = PutObjReader::new(hash_reader);
|
||||
return match self_.clone().put_object(bucket, object, &mut p_reader, &ropts).await {
|
||||
Ok(restored_info) => {
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, &opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
@@ -4395,7 +4210,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return set_restore_header_fn(&mut oi, Some(err)).await;
|
||||
}
|
||||
};
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
@@ -4854,33 +4668,6 @@ mod transition_commit_failure_tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_unsupported_remote_versions_are_cleaned_up() {
|
||||
for remote_version in ["null", "opaque-version-token"] {
|
||||
let manager = TierConfigMgr::new();
|
||||
let backend = register_mock_tier(&manager, "WARM").await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&manager, "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
let candidate = TransitionUploadCandidate::from_put_response(remote_version.to_string());
|
||||
|
||||
persisted_transition_version(candidate.remote_version()).expect_err("unsupported writer version must fail closed");
|
||||
cleanup_rejected_transition_upload_durably(
|
||||
&lease,
|
||||
"remote/object",
|
||||
candidate.cleanup_version(),
|
||||
candidate.cleanup_version_is_exact(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("rejected remote upload must be cleaned up");
|
||||
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), candidate.cleanup_version().to_string())]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[serial_test::serial(restore_multipart_failure_point)]
|
||||
async fn multipart_restore_aborts_every_post_create_failure() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -6828,45 +6615,6 @@ mod transition_upload_integrity_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn unversioned_remote_version_is_persisted_without_version_id() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transition-unversioned-tier-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"unversioned remote tier must commit without a version id".repeat(1024);
|
||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
let save_probe = TransitionUploadedSaveProbe::install(bucket, object);
|
||||
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect("an unversioned remote version must commit");
|
||||
let (fi, _, _) = set_disks
|
||||
.get_object_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("committed unversioned transition metadata should be readable");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
assert_eq!(fi.transition_version_state, rustfs_filemeta::TransitionVersionState::KnownDisabled);
|
||||
assert_eq!(save_probe.attempts(), 1);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn opaque_remote_version_is_cleaned_before_parse_failure() {
|
||||
@@ -6882,7 +6630,7 @@ mod transition_upload_integrity_tests {
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect_err("an opaque remote version must fail closed until the capability gate is active");
|
||||
.expect_err("an unparseable remote version must fail closed");
|
||||
let removed_versions = backend.remove_versions().await;
|
||||
assert_eq!(removed_versions.len(), 1);
|
||||
assert_eq!(removed_versions[0].1, "opaque-version-token");
|
||||
@@ -6890,38 +6638,6 @@ mod transition_upload_integrity_tests {
|
||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn nil_remote_version_is_cleaned_exactly_before_transaction_persistence() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transition-nil-version-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"nil remote version must retain local data".repeat(1024);
|
||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let remote_version = Uuid::nil().to_string();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
let save_probe = TransitionUploadedSaveProbe::install(bucket, object);
|
||||
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect_err("a nil remote version must fail closed before transaction persistence");
|
||||
let put_versions = backend.put_versions().await;
|
||||
let removed_versions = backend.remove_versions().await;
|
||||
assert_eq!(removed_versions, put_versions);
|
||||
assert_eq!(removed_versions.len(), 1);
|
||||
assert_eq!(
|
||||
removed_versions.first().map(|(_, version)| version.as_str()),
|
||||
Some(remote_version.as_str())
|
||||
);
|
||||
assert_eq!(save_probe.attempts(), 0, "nil remote version must be rejected before saving Uploaded");
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn authoritative_read_failure_after_upload_cleans_exact_candidate_and_preserves_source() {
|
||||
@@ -7241,6 +6957,7 @@ mod transition_source_identity_matrix_tests {
|
||||
async fn transition_source_identity_field_matrix_rejects_single_field_drift() {
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum IdentityField {
|
||||
VersionId,
|
||||
DataDir,
|
||||
ModTime,
|
||||
Size,
|
||||
@@ -7256,6 +6973,7 @@ mod transition_source_identity_matrix_tests {
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
|
||||
for (index, field) in [
|
||||
IdentityField::VersionId,
|
||||
IdentityField::DataDir,
|
||||
IdentityField::ModTime,
|
||||
IdentityField::Size,
|
||||
@@ -7267,36 +6985,23 @@ mod transition_source_identity_matrix_tests {
|
||||
let object = format!("identity-{index}.bin");
|
||||
let payload = vec![u8::try_from(index + 1).expect("matrix index should fit u8"); 1024 * 1024];
|
||||
let mut reader = PutObjReader::from_vec(payload);
|
||||
let source_version_id = Uuid::new_v4();
|
||||
let source_opts = ObjectOptions {
|
||||
version_id: Some(source_version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let original = set_disks
|
||||
.put_object(bucket, &object, &mut reader, &source_opts)
|
||||
.put_object(bucket, &object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let (source, _, _) = set_disks
|
||||
.get_object_fileinfo(bucket, &object, &source_opts, true, false)
|
||||
.get_object_fileinfo(bucket, &object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("source metadata should resolve");
|
||||
assert_eq!(source.version_id, Some(source_version_id));
|
||||
assert_eq!(
|
||||
transition_source_identity(bucket, &object, &source, &source_opts, &get_raw_etag(&source.metadata))
|
||||
.expect("persisted versioned source identity should build")
|
||||
.version_mode,
|
||||
TransitionSourceVersionMode::Versioned
|
||||
);
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.clone(),
|
||||
etag: original.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
@@ -7309,6 +7014,7 @@ mod transition_source_identity_matrix_tests {
|
||||
|
||||
let mut changed = source.clone();
|
||||
match field {
|
||||
IdentityField::VersionId => changed.version_id = Some(Uuid::new_v4()),
|
||||
IdentityField::DataDir => changed.data_dir = Some(Uuid::new_v4()),
|
||||
IdentityField::ModTime => {
|
||||
changed.mod_time = changed.mod_time.map(|value| value + time::Duration::nanoseconds(1));
|
||||
@@ -7326,19 +7032,12 @@ mod transition_source_identity_matrix_tests {
|
||||
.await
|
||||
.expect("single-field metadata drift should be written");
|
||||
}
|
||||
let persisted_opts = ObjectOptions {
|
||||
version_id: changed.version_id.map(|version_id| version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (persisted, _, _) = set_disks
|
||||
.get_object_fileinfo(bucket, &object, &persisted_opts, true, false)
|
||||
.await
|
||||
.expect("drifted source metadata should resolve");
|
||||
put_barrier.release();
|
||||
|
||||
let result = transition.await.expect("transition task should not panic");
|
||||
assert!(result.is_err(), "transition must reject {field:?} drift");
|
||||
transition
|
||||
.await
|
||||
.expect("transition task should not panic")
|
||||
.expect_err("transition must reject a source whose identity changed after upload");
|
||||
let expected_attempts = index + 1;
|
||||
assert_eq!(backend.put_count().await, expected_attempts);
|
||||
assert_eq!(backend.remove_count().await, expected_attempts);
|
||||
@@ -7349,23 +7048,26 @@ mod transition_source_identity_matrix_tests {
|
||||
);
|
||||
|
||||
match field {
|
||||
IdentityField::DataDir => assert_ne!(source.data_dir, persisted.data_dir),
|
||||
IdentityField::ModTime => assert_ne!(source.mod_time, persisted.mod_time),
|
||||
IdentityField::Size => assert_ne!(source.size, persisted.size),
|
||||
IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&persisted.metadata)),
|
||||
IdentityField::VersionId => assert_ne!(source.version_id, changed.version_id),
|
||||
IdentityField::DataDir => assert_ne!(source.data_dir, changed.data_dir),
|
||||
IdentityField::ModTime => assert_ne!(source.mod_time, changed.mod_time),
|
||||
IdentityField::Size => assert_ne!(source.size, changed.size),
|
||||
IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&changed.metadata)),
|
||||
}
|
||||
if !matches!(field, IdentityField::VersionId) {
|
||||
assert_eq!(source.version_id, changed.version_id);
|
||||
}
|
||||
assert_eq!(source.version_id, persisted.version_id);
|
||||
if !matches!(field, IdentityField::DataDir) {
|
||||
assert_eq!(source.data_dir, persisted.data_dir);
|
||||
assert_eq!(source.data_dir, changed.data_dir);
|
||||
}
|
||||
if !matches!(field, IdentityField::ModTime) {
|
||||
assert_eq!(source.mod_time, persisted.mod_time);
|
||||
assert_eq!(source.mod_time, changed.mod_time);
|
||||
}
|
||||
if !matches!(field, IdentityField::Size) {
|
||||
assert_eq!(source.size, persisted.size);
|
||||
assert_eq!(source.size, changed.size);
|
||||
}
|
||||
if !matches!(field, IdentityField::Etag) {
|
||||
assert_eq!(get_raw_etag(&source.metadata), get_raw_etag(&persisted.metadata));
|
||||
assert_eq!(get_raw_etag(&source.metadata), get_raw_etag(&changed.metadata));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1971,6 +1971,17 @@ mod metadata_cache_tests {
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.erasure.index = fi.erasure.distribution[disk_index];
|
||||
// `valid_test_fileinfo` carries a positive size with no parts —
|
||||
// the shape `validate_collection_contents` rejects as
|
||||
// `FileCorrupt` — so it can drive
|
||||
// `get_object_with_fileinfo_rejects_positive_size_without_parts`.
|
||||
// Metadata that has to survive a write needs the matching part.
|
||||
fi.parts.push(ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
});
|
||||
disk.write_metadata(bucket, bucket, object, fi)
|
||||
.await
|
||||
.expect("metadata should be written before quorum read");
|
||||
|
||||
@@ -13,10 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::lifecycle;
|
||||
use rustfs_filemeta::RestoreStatusOps;
|
||||
use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
|
||||
use s3s::dto::{RestoreStatus, Timestamp};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct RestoreCleanupIdentity {
|
||||
@@ -46,69 +43,6 @@ impl RestoreCleanupIdentity {
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn finalize_restore_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
obj_info: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let expected = RestoreCleanupIdentity::from_object_info(obj_info);
|
||||
let expected_operation_id = restore_operation_id_from_metadata(&opts.user_defined)?;
|
||||
let expected_etag = obj_info
|
||||
.etag
|
||||
.clone()
|
||||
.unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref()));
|
||||
let version_id = expected.version_id.map(|v| v.to_string());
|
||||
let _lock_guard = if !opts.no_lock {
|
||||
Some(
|
||||
self.acquire_write_lock_diag("restore_finalize_metadata", bucket, object)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let read_opts = ObjectOptions {
|
||||
version_id,
|
||||
versioned: opts.versioned,
|
||||
version_suspended: opts.version_suspended,
|
||||
..Default::default()
|
||||
};
|
||||
let (mut fi, _, disks) = self
|
||||
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
|
||||
.await?;
|
||||
if let Some(expected_operation_id) = expected_operation_id {
|
||||
require_restore_operation_id(&fi.metadata, expected_operation_id)?;
|
||||
}
|
||||
if !expected.matches_file_info(&fi, &expected_etag) {
|
||||
return Err(Error::other("restored object changed before restore metadata finalization"));
|
||||
}
|
||||
let restore_expiry =
|
||||
lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
|
||||
fi.metadata.insert(
|
||||
X_AMZ_RESTORE.as_str().to_string(),
|
||||
RestoreStatus {
|
||||
is_restore_in_progress: Some(false),
|
||||
restore_expiry_date: Some(Timestamp::from(restore_expiry)),
|
||||
}
|
||||
.to_string(),
|
||||
);
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
self.update_object_meta_with_opts(
|
||||
bucket,
|
||||
object,
|
||||
fi.clone(),
|
||||
disks.as_slice(),
|
||||
&UpdateMetadataOpts {
|
||||
replace_user_metadata: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
|
||||
}
|
||||
|
||||
pub async fn update_restore_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -13,11 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
|
||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions};
|
||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
async fn prime_metadata_generation(set_disks: &SetDisks, bucket: &str, object: &str) -> GetObjectMetadataCacheKey {
|
||||
@@ -84,57 +83,13 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
|
||||
let transitioned_generation = prime_metadata_generation(&set_disks, bucket, object).await;
|
||||
let mut restore_opts = ObjectOptions::default();
|
||||
restore_opts.transition.restore_request.days = Some(1);
|
||||
let restore_started = OffsetDateTime::now_utc();
|
||||
let expiry_from_restore_start = temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_ILM_DEBUG_DAY_SECS", Some("1")),
|
||||
("RUSTFS_ILM_PROCESS_TIME", Some("1")),
|
||||
],
|
||||
async {
|
||||
let expiry_from_restore_start = expected_expiry_time(restore_started, 1);
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
let restore_set = Arc::clone(&set_disks);
|
||||
let restore =
|
||||
tokio::spawn(async move { restore_set.restore_transitioned_object(bucket, object, &restore_opts).await });
|
||||
get_barrier.wait_until_paused().await;
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if expected_expiry_time(OffsetDateTime::now_utc(), 1) > expiry_from_restore_start {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("test clock should cross the next accelerated lifecycle boundary");
|
||||
get_barrier.release();
|
||||
restore
|
||||
.await
|
||||
.expect("restore task should join")
|
||||
.expect("restore should succeed");
|
||||
expiry_from_restore_start
|
||||
},
|
||||
)
|
||||
.await;
|
||||
Arc::clone(&set_disks)
|
||||
.restore_transitioned_object(bucket, object, &restore_opts)
|
||||
.await
|
||||
.expect("restore should succeed");
|
||||
assert_generation_reclaimed(&set_disks, &transitioned_generation).await;
|
||||
assert_eq!(backend.get_count().await, 1, "restore should read the remote candidate exactly once");
|
||||
|
||||
let restored_info = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("restored object metadata should be readable");
|
||||
let restore_status = parse_restore_obj_status(
|
||||
restored_info
|
||||
.user_defined
|
||||
.get(s3s::header::X_AMZ_RESTORE.as_str())
|
||||
.expect("completed restore header should be present"),
|
||||
)
|
||||
.expect("completed restore header should parse");
|
||||
assert!(
|
||||
restore_status.expiry().expect("completed restore should have an expiry") > expiry_from_restore_start,
|
||||
"restore expiry must be based on completion, not the time the remote copy started"
|
||||
);
|
||||
|
||||
let mut restored = Vec::new();
|
||||
set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
|
||||
@@ -87,7 +87,7 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
|
||||
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
|
||||
}
|
||||
|
||||
pub(crate) async fn await_bucket_namespace_operation<T, F>(
|
||||
async fn await_bucket_namespace_operation<T, F>(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &str,
|
||||
operation: &'static str,
|
||||
|
||||
@@ -1311,16 +1311,14 @@ mod tests {
|
||||
version_id: "version-a".to_string(),
|
||||
tier_name: tier_a.to_string(),
|
||||
backend_identity: Some(identity_a),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
let entry_b = Jentry {
|
||||
obj_name: "remote-b".to_string(),
|
||||
version_id: "version-b".to_string(),
|
||||
tier_name: tier_b.to_string(),
|
||||
backend_identity: Some(identity_b),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
let remove_a = backend_a.arm_failing_remove_barrier().await;
|
||||
persist_tier_delete_journal_entry(store_a.clone(), &entry_a)
|
||||
@@ -2722,12 +2720,10 @@ mod tests {
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transition_transaction_recovery_deletes_provider_recovered_unknown_upload() {
|
||||
let versioned_remote = uuid::Uuid::new_v4().to_string();
|
||||
let nil_remote = uuid::Uuid::nil().to_string();
|
||||
for (case, tier_name, remote_version) in [
|
||||
("missing", "TXPROBEMISSING", None),
|
||||
("unversioned", "TXPROBEUNVERSIONED", Some(String::new())),
|
||||
("versioned", "TXPROBEVERSIONED", Some(versioned_remote)),
|
||||
("nil-version", "TXPROBENILVERSION", Some(nil_remote)),
|
||||
] {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
|
||||
|
||||
@@ -141,7 +141,6 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
|
||||
const MAX_UPLOADS_LIST: usize = 10000;
|
||||
|
||||
mod bucket;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
mod heal;
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
|
||||
@@ -219,16 +219,6 @@ impl ErasureInfo {
|
||||
}
|
||||
|
||||
// #[derive(Debug, Clone)]
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy, Default)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum TransitionVersionState {
|
||||
#[default]
|
||||
Unknown,
|
||||
KnownDisabled,
|
||||
SuspendedNull,
|
||||
Exact,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
|
||||
pub struct FileInfo {
|
||||
pub volume: String,
|
||||
@@ -240,10 +230,6 @@ pub struct FileInfo {
|
||||
pub transitioned_objname: String,
|
||||
pub transition_tier: String,
|
||||
pub transition_version_id: Option<Uuid>,
|
||||
#[serde(default)]
|
||||
pub transition_version: Option<String>,
|
||||
#[serde(default)]
|
||||
pub transition_version_state: TransitionVersionState,
|
||||
pub expire_restored: bool,
|
||||
pub data_dir: Option<Uuid>,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
@@ -473,10 +459,6 @@ impl FileInfo {
|
||||
if self.mod_time.is_none_or(|mod_time| mod_time <= OffsetDateTime::UNIX_EPOCH)
|
||||
|| (!allow_nil_version_id && self.version_id.is_some_and(|version_id| version_id.is_nil()))
|
||||
|| self.transition_version_id.is_some_and(|version_id| version_id.is_nil())
|
||||
|| self
|
||||
.transition_version
|
||||
.as_ref()
|
||||
.is_some_and(|version_id| version_id.is_empty())
|
||||
|| self.size != 0
|
||||
|| self.data_dir.is_some()
|
||||
|| self.mode.is_some()
|
||||
@@ -510,7 +492,6 @@ impl FileInfo {
|
||||
|| !self.transitioned_objname.is_empty()
|
||||
|| !self.transition_tier.is_empty()
|
||||
|| self.transition_version_id.is_some()
|
||||
|| self.transition_version.is_some()
|
||||
|| self.expire_restored
|
||||
|| self.size != 0
|
||||
|| self.data_dir.is_some()
|
||||
@@ -555,25 +536,6 @@ impl FileInfo {
|
||||
/// return `None`.
|
||||
pub fn validate(&self, mode: ValidationMode) -> Result<Option<ValidatedErasureLayout>> {
|
||||
self.validate_collection_bounds()?;
|
||||
if let (Some(version), Some(version_id)) = (&self.transition_version, self.transition_version_id)
|
||||
&& Uuid::parse_str(version).ok() != Some(version_id)
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let transition_state_valid = match self.transition_version_state {
|
||||
TransitionVersionState::Unknown => true,
|
||||
TransitionVersionState::KnownDisabled => self.transition_version.is_none() && self.transition_version_id.is_none(),
|
||||
TransitionVersionState::SuspendedNull => {
|
||||
self.transition_version.as_deref() == Some("null") && self.transition_version_id.is_none()
|
||||
}
|
||||
TransitionVersionState::Exact => self
|
||||
.transition_version
|
||||
.as_deref()
|
||||
.is_some_and(|version| version != "null" && !version.is_empty()),
|
||||
};
|
||||
if !transition_state_valid {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
|
||||
let erasure_layout = match mode {
|
||||
ValidationMode::RequireErasure => Some(self.validate_erasure_geometry()?),
|
||||
@@ -870,8 +832,6 @@ impl FileInfo {
|
||||
&& self.transition_tier == other.transition_tier
|
||||
&& self.transitioned_objname == other.transitioned_objname
|
||||
&& self.transition_version_id == other.transition_version_id
|
||||
&& self.transition_version == other.transition_version
|
||||
&& self.transition_version_state == other.transition_version_state
|
||||
}
|
||||
|
||||
/// Check if metadata maps are equal
|
||||
@@ -1391,15 +1351,6 @@ mod tests {
|
||||
assert_file_corrupt(&fi, ValidationMode::DeleteOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_read_validation_rejects_conflicting_transition_versions() {
|
||||
let mut fi = one_shard_validation_fileinfo(1);
|
||||
fi.transition_version_id = Some(Uuid::new_v4());
|
||||
fi.transition_version = Some(Uuid::new_v4().to_string());
|
||||
|
||||
assert_file_corrupt(&fi, ValidationMode::RequireErasure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_read_validation_requires_canonical_delete_marker_shape() {
|
||||
let marker = FileInfo {
|
||||
@@ -1771,12 +1722,6 @@ mod tests {
|
||||
transitioned_objname,
|
||||
transition_tier,
|
||||
transition_version_id,
|
||||
transition_version: transition_version_id.map(|version_id| version_id.to_string()),
|
||||
transition_version_state: if transition_version_id.is_some() {
|
||||
TransitionVersionState::Exact
|
||||
} else {
|
||||
TransitionVersionState::Unknown
|
||||
},
|
||||
expire_restored,
|
||||
data_dir,
|
||||
mod_time,
|
||||
|
||||
@@ -356,14 +356,6 @@ impl FileMeta {
|
||||
.versions
|
||||
.partition_point(|existing| existing.header.sorts_before(&new_shallow.header));
|
||||
self.versions.insert(insert_pos, new_shallow);
|
||||
// `partition_point` only returns the canonical slot when `versions` is
|
||||
// already canonically ordered, and nothing establishes that: `FileMeta::load`
|
||||
// replays the on-disk order verbatim, and metadata written before the
|
||||
// canonical-order fix ordered equal-`mod_time` ties by insertion rather than
|
||||
// by `sorts_before`. Re-sort so an insert leaves the list canonical either
|
||||
// way, matching what `set_idx` already does on the replace path. Cheap: after
|
||||
// a correctly placed insert the slice is a single sorted run.
|
||||
self.sort_by_mod_time();
|
||||
Ok(())
|
||||
|
||||
// if !ver.valid() {
|
||||
@@ -1223,57 +1215,6 @@ mod test {
|
||||
assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header));
|
||||
}
|
||||
|
||||
/// `add_version_filemata` positions an inserted version with
|
||||
/// `partition_point(sorts_before)`, which only yields the canonical slot when
|
||||
/// `versions` is already canonically ordered. Nothing establishes that
|
||||
/// precondition on load: `FileMeta::unmarshal_msg` pushes versions in file
|
||||
/// order, and metadata written before the canonical-order fix ordered
|
||||
/// equal-`mod_time` ties by insertion instead of by `sorts_before`. An insert
|
||||
/// into such a list must still leave it canonical, the same way `set_idx`
|
||||
/// already re-sorts on the replace path.
|
||||
#[test]
|
||||
fn add_version_filemata_canonicalizes_versions_loaded_out_of_order() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
let first = version_for_ordering(VersionType::Object, Uuid::from_u128(70), mod_time, 70);
|
||||
let second = version_for_ordering(VersionType::Object, Uuid::from_u128(80), mod_time, 80);
|
||||
let (early, late) = if first.header().sorts_before(&second.header()) {
|
||||
(first, second)
|
||||
} else {
|
||||
(second, first)
|
||||
};
|
||||
|
||||
// Persist the pair the wrong way round to emulate a pre-canonical-order
|
||||
// xl.meta, bypassing add_version_filemata so the bad order really lands on
|
||||
// the wire.
|
||||
let mut legacy = FileMeta::new();
|
||||
legacy
|
||||
.versions
|
||||
.push(FileMetaShallowVersion::try_from(late).expect("shallow later version"));
|
||||
legacy
|
||||
.versions
|
||||
.push(FileMetaShallowVersion::try_from(early).expect("shallow earlier version"));
|
||||
|
||||
let mut loaded =
|
||||
FileMeta::load(&legacy.marshal_msg().expect("serialize legacy metadata")).expect("reload legacy metadata");
|
||||
assert!(
|
||||
!loaded.versions[0].header.sorts_before(&loaded.versions[1].header),
|
||||
"fixture must reach add_version_filemata non-canonically ordered"
|
||||
);
|
||||
|
||||
loaded
|
||||
.add_version_filemata(version_for_ordering(VersionType::Delete, Uuid::from_u128(90), mod_time, 90))
|
||||
.expect("insert equal-time delete marker");
|
||||
|
||||
assert_eq!(loaded.versions.len(), 3);
|
||||
assert!(
|
||||
loaded
|
||||
.versions
|
||||
.windows(2)
|
||||
.all(|pair| pair[0].header.sorts_before(&pair[1].header)),
|
||||
"insert must leave the version list canonically ordered"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression for backlog#799 B16: replication reset state must be
|
||||
/// persisted under both internal prefixes, never as a bare ARN. A bare-ARN
|
||||
/// key (produced by `ObjectInfo::replication_state`) has no internal prefix,
|
||||
|
||||
@@ -26,14 +26,13 @@ use super::msgp_decode::{
|
||||
PrependByteReader, prealloc_hint, read_exact_vec, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value,
|
||||
};
|
||||
use super::*;
|
||||
use crate::{ChecksumInfo, TransitionVersionState};
|
||||
use crate::ChecksumInfo;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use rustfs_utils::http::{
|
||||
RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID,
|
||||
SUFFIX_TIER_FV_MARKER, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
|
||||
get_bytes, get_consistent_bytes, get_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes,
|
||||
strip_internal_prefix,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, contains_key_bytes, get_bytes, get_consistent_bytes, get_str,
|
||||
has_internal_suffix, insert_bytes, is_internal_key, remove_bytes, strip_internal_prefix,
|
||||
};
|
||||
|
||||
const MSGPACK_EXT8: u8 = 0xc7;
|
||||
@@ -44,7 +43,6 @@ const MSGPACK_FIXEXT8: u8 = 0xd7;
|
||||
const MSGPACK_TIME_EXT_LEGACY: i8 = 5;
|
||||
const MSGPACK_TIME_EXT_OFFICIAL: i8 = -1;
|
||||
const MSGPACK_TIME_LEN: u8 = 12;
|
||||
const MAX_TRANSITION_VERSION_LEN: usize = 1024;
|
||||
|
||||
/// Sentinel signature returned when a version has no computable body (invalid /
|
||||
/// missing inner object). Mirrors MinIO's `signatureErr` so such versions never
|
||||
@@ -253,93 +251,23 @@ fn parse_legacy_uuid_bytes(bytes: &[u8], field: &str) -> Result<Option<Uuid>> {
|
||||
|
||||
/// Decode a stored transitioned-version-id from a version's `meta_sys`.
|
||||
///
|
||||
/// Legacy RustFS writes used 16 raw UUID bytes. New writes and MinIO-migrated
|
||||
/// records use the provider's exact UTF-8 version text. Empty, nil UUID, and
|
||||
/// malformed bytes are not usable remote versions.
|
||||
fn transitioned_version_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Result<Option<String>> {
|
||||
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(value) = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value = value.to_vec();
|
||||
/// RustFS writes it as 16 raw UUID bytes; MinIO-migrated tiered objects store
|
||||
/// the remote tier's version id as a UUID *string*. Accept both, and treat any
|
||||
/// absent / nil / otherwise-unparseable value as "no tier version" (matching the
|
||||
/// tolerant pre-hardening behavior) rather than failing the whole object read —
|
||||
/// a malformed tier id must not make an otherwise-readable object unreadable.
|
||||
fn transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
|
||||
let value = get_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)?;
|
||||
if value.is_empty() {
|
||||
return Ok(None);
|
||||
return None;
|
||||
}
|
||||
if let Ok(id) = Uuid::from_slice(&value) {
|
||||
return Ok((!id.is_nil()).then(|| id.to_string()));
|
||||
return (!id.is_nil()).then_some(id);
|
||||
}
|
||||
let Ok(value) = String::from_utf8(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if value.is_empty()
|
||||
|| value.len() > MAX_TRANSITION_VERSION_LEN
|
||||
|| value.chars().any(char::is_control)
|
||||
|| Uuid::parse_str(&value).is_ok_and(|id| id.is_nil())
|
||||
{
|
||||
Ok(None)
|
||||
} else {
|
||||
Ok(Some(value))
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_version_state_from_meta_sys(
|
||||
meta_sys: &HashMap<String, Vec<u8>>,
|
||||
version: Option<&str>,
|
||||
) -> Result<TransitionVersionState> {
|
||||
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
|
||||
return Ok(TransitionVersionState::Unknown);
|
||||
}
|
||||
let value = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or(Error::FileCorrupt)?;
|
||||
let state = match value {
|
||||
b"known-disabled" => TransitionVersionState::KnownDisabled,
|
||||
b"suspended-null" => TransitionVersionState::SuspendedNull,
|
||||
b"exact" => TransitionVersionState::Exact,
|
||||
b"unknown" => TransitionVersionState::Unknown,
|
||||
_ => return Err(Error::FileCorrupt),
|
||||
};
|
||||
let valid = match state {
|
||||
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
|
||||
TransitionVersionState::SuspendedNull => version == Some("null"),
|
||||
TransitionVersionState::Exact => version.is_some_and(|value| value != "null"),
|
||||
};
|
||||
valid.then_some(state).ok_or(Error::FileCorrupt)
|
||||
}
|
||||
|
||||
fn transition_version_state_bytes(state: TransitionVersionState) -> &'static [u8] {
|
||||
match state {
|
||||
TransitionVersionState::Unknown => b"unknown",
|
||||
TransitionVersionState::KnownDisabled => b"known-disabled",
|
||||
TransitionVersionState::SuspendedNull => b"suspended-null",
|
||||
TransitionVersionState::Exact => b"exact",
|
||||
}
|
||||
}
|
||||
|
||||
fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: TransitionVersionState) {
|
||||
if state == TransitionVersionState::Unknown {
|
||||
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
|
||||
} else {
|
||||
insert_bytes(
|
||||
meta_sys,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
transition_version_state_bytes(state).to_vec(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn legacy_transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
|
||||
transitioned_version_from_meta_sys(meta_sys)
|
||||
std::str::from_utf8(&value)
|
||||
.ok()
|
||||
.flatten()
|
||||
.and_then(|value| Uuid::parse_str(&value).ok())
|
||||
}
|
||||
|
||||
fn transitioned_version_bytes(fi: &FileInfo) -> Option<Vec<u8>> {
|
||||
fi.transition_version
|
||||
.as_ref()
|
||||
.map(|version| version.as_bytes().to_vec())
|
||||
.or_else(|| fi.transition_version_id.map(|version_id| version_id.as_bytes().to_vec()))
|
||||
.and_then(|s| Uuid::parse_str(s.trim()).ok())
|
||||
.filter(|id| !id.is_nil())
|
||||
}
|
||||
|
||||
fn parse_legacy_erasure_algo(value: &str) -> ErasureAlgo {
|
||||
@@ -2470,9 +2398,7 @@ impl MetaObject {
|
||||
let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
|
||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
.unwrap_or_default();
|
||||
let transition_version = transitioned_version_from_meta_sys(&self.meta_sys)?;
|
||||
let transition_version_state = transition_version_state_from_meta_sys(&self.meta_sys, transition_version.as_deref())?;
|
||||
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
||||
let transition_version_id = transitioned_version_id_from_meta_sys(&self.meta_sys);
|
||||
let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
|
||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
.unwrap_or_default();
|
||||
@@ -2493,8 +2419,6 @@ impl MetaObject {
|
||||
transition_status,
|
||||
transitioned_objname,
|
||||
transition_version_id,
|
||||
transition_version,
|
||||
transition_version_state,
|
||||
transition_tier,
|
||||
..Default::default()
|
||||
})
|
||||
@@ -2507,12 +2431,13 @@ impl MetaObject {
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
fi.transitioned_objname.as_bytes().to_vec(),
|
||||
);
|
||||
if let Some(transition_version) = transitioned_version_bytes(fi) {
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
} else {
|
||||
remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
|
||||
if let Some(transition_version_id) = fi.transition_version_id.as_ref() {
|
||||
insert_bytes(
|
||||
&mut self.meta_sys,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
transition_version_id.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state);
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
|
||||
if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes());
|
||||
@@ -2576,7 +2501,6 @@ impl MetaObject {
|
||||
SUFFIX_TRANSITION_TIER,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
] {
|
||||
if let Some(v) = get_bytes(&self.meta_sys, suffix) {
|
||||
insert_bytes(&mut delete_marker.meta_sys, suffix, v);
|
||||
@@ -2638,11 +2562,8 @@ impl From<FileInfo> for MetaObject {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(transition_version) = transitioned_version_bytes(&value) {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
if let Some(vid) = &value.transition_version_id {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, vid.as_bytes().to_vec());
|
||||
}
|
||||
|
||||
if !value.transition_tier.is_empty() {
|
||||
@@ -2785,11 +2706,7 @@ impl MetaDeleteMarker {
|
||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
fi.transition_version = transitioned_version_from_meta_sys(&self.meta_sys).ok().flatten();
|
||||
fi.transition_version_id = legacy_transitioned_version_id_from_meta_sys(&self.meta_sys);
|
||||
fi.transition_version_state =
|
||||
transition_version_state_from_meta_sys(&self.meta_sys, fi.transition_version.as_deref())
|
||||
.unwrap_or(TransitionVersionState::Unknown);
|
||||
fi.transition_version_id = transitioned_version_id_from_meta_sys(&self.meta_sys);
|
||||
}
|
||||
|
||||
fi
|
||||
@@ -2942,11 +2859,8 @@ impl From<FileInfo> for MetaDeleteMarker {
|
||||
value.transitioned_objname.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
if let Some(transition_version) = transitioned_version_bytes(&value) {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() || value.tier_free_version() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
if let Some(version_id) = value.transition_version_id {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, version_id.as_bytes().to_vec());
|
||||
}
|
||||
if !value.transition_tier.is_empty() {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
|
||||
@@ -3498,7 +3412,7 @@ mod tests {
|
||||
.insert("x-rustfs-internal-healing".to_string(), "true".to_string());
|
||||
marker.metadata.insert("content-type".to_string(), "text/plain".to_string());
|
||||
let remote_version_id = Uuid::new_v4();
|
||||
marker.transition_version = Some(remote_version_id.to_string());
|
||||
marker.transition_version_id = Some(remote_version_id);
|
||||
|
||||
let converted = MetaDeleteMarker::from(marker);
|
||||
|
||||
@@ -3506,19 +3420,7 @@ mod tests {
|
||||
assert_eq!(converted.meta_sys.get("x-minio-internal-purgestatus"), Some(&b"pending".to_vec()));
|
||||
assert_eq!(
|
||||
get_bytes(&converted.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID),
|
||||
Some(remote_version_id.to_string().into_bytes())
|
||||
);
|
||||
assert_eq!(
|
||||
converted
|
||||
.meta_sys
|
||||
.get(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}")),
|
||||
Some(&remote_version_id.to_string().into_bytes())
|
||||
);
|
||||
assert_eq!(
|
||||
converted
|
||||
.meta_sys
|
||||
.get(&format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)),
|
||||
Some(&remote_version_id.to_string().into_bytes())
|
||||
Some(remote_version_id.as_bytes().to_vec())
|
||||
);
|
||||
assert!(!converted.meta_sys.contains_key("x-rustfs-internal-healing"));
|
||||
assert!(!converted.meta_sys.contains_key("content-type"));
|
||||
@@ -4195,129 +4097,19 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("into_fileinfo");
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_id_opaque_text_is_preserved() {
|
||||
fn meta_object_transition_version_id_unparseable_stays_readable_as_none() {
|
||||
// A non-UUID / non-16-byte tier version id must NOT make the object
|
||||
// unreadable; it is tolerated as "no tier version" (compat with
|
||||
// pre-hardening behavior and foreign/edge metadata).
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"opaque-generation-42".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"not-a-uuid".to_vec());
|
||||
let fi = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("opaque transition version id must decode");
|
||||
.expect("unparseable transition version id must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_exact_round_trips_dual_keys() {
|
||||
let id = sample_version_id();
|
||||
let expected_version = id.to_string();
|
||||
let fi = FileInfo {
|
||||
transition_status: "complete".to_string(),
|
||||
transition_version: Some(expected_version.clone()),
|
||||
transition_version_state: TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let object = MetaObject::from(fi);
|
||||
assert_eq!(
|
||||
object
|
||||
.meta_sys
|
||||
.get(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"))
|
||||
.map(Vec::as_slice),
|
||||
Some(b"exact".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
object
|
||||
.meta_sys
|
||||
.get(&format!(
|
||||
"{}{SUFFIX_TRANSITIONED_VERSION_STATE}",
|
||||
rustfs_utils::http::MINIO_INTERNAL_PREFIX
|
||||
))
|
||||
.map(Vec::as_slice),
|
||||
Some(b"exact".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
legacy_transitioned_version_id_from_meta_sys(&object.meta_sys),
|
||||
Some(id),
|
||||
"UUID exact writes must remain readable by the legacy UUID consumer"
|
||||
);
|
||||
let decoded = object.into_fileinfo("b", "k", false).expect("exact state should round trip");
|
||||
assert_eq!(decoded.transition_version_state, TransitionVersionState::Exact);
|
||||
assert_eq!(decoded.transition_version.as_deref(), Some(expected_version.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_transition_known_disabled_removes_stale_version_dual_keys() {
|
||||
let mut meta_sys = HashMap::new();
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, b"stale-legacy-version".to_vec());
|
||||
let mut object = make_meta_object_with_sys(meta_sys);
|
||||
object.set_transition(&FileInfo {
|
||||
transition_status: TRANSITION_COMPLETE.to_string(),
|
||||
transitioned_objname: "remote/object".to_string(),
|
||||
transition_version_state: TransitionVersionState::KnownDisabled,
|
||||
transition_tier: "WARM".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(get_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID), None);
|
||||
assert!(
|
||||
!object
|
||||
.meta_sys
|
||||
.contains_key(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}"))
|
||||
);
|
||||
assert!(
|
||||
!object
|
||||
.meta_sys
|
||||
.contains_key(&format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX))
|
||||
);
|
||||
let decoded = object
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("known-disabled transition must remain readable after replacing stale metadata");
|
||||
assert_eq!(decoded.transition_version, None);
|
||||
assert_eq!(decoded.transition_version_state, TransitionVersionState::KnownDisabled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_conflict_fails_closed() {
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, sample_version_id().as_bytes().to_vec());
|
||||
sys.insert(format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"), b"exact".to_vec());
|
||||
sys.insert(
|
||||
format!("{}{SUFFIX_TRANSITIONED_VERSION_STATE}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
|
||||
b"known-disabled".to_vec(),
|
||||
);
|
||||
|
||||
make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect_err("conflicting state keys must fail closed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_id_invalid_utf8_yields_none() {
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, vec![0xff]);
|
||||
let fi = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("invalid transition version bytes must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_id_unsafe_text_yields_none() {
|
||||
for value in [b"opaque\0version".to_vec(), vec![b'x'; MAX_TRANSITION_VERSION_LEN + 1]] {
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, value);
|
||||
let fi = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("unsafe transition version text must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4331,7 +4123,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("string-form transition version id must decode");
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4361,14 +4152,16 @@ mod tests {
|
||||
}
|
||||
.into_fileinfo("b", "k", false);
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_free_version_transition_version_id_opaque_text_is_preserved() {
|
||||
fn delete_marker_free_version_transition_version_id_unparseable_stays_readable() {
|
||||
// A malformed tier version id must not make a free-version record corrupt:
|
||||
// it decodes to None and stays readable. Otherwise free-version expiry
|
||||
// fails and the remote-tier object leaks.
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"opaque-generation-42".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"not-a-uuid".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITION_TIER, b"WARM".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_OBJECTNAME, b"remote-object".to_vec());
|
||||
let fi = MetaDeleteMarker {
|
||||
@@ -4379,9 +4172,8 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false);
|
||||
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
|
||||
fi.validate_for_metadata_read()
|
||||
.expect("free-version record with an opaque tier id must remain readable");
|
||||
.expect("free-version record with an unparseable tier id must remain readable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4401,7 +4193,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false);
|
||||
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -26,9 +26,6 @@ documentation = "https://docs.rs/rustfs-heal/latest/rustfs_heal/"
|
||||
keywords = ["RustFS", "heal", "erasure-coding", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "filesystem"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true }
|
||||
rustfs-concurrency = { workspace = true }
|
||||
@@ -56,7 +53,7 @@ serial_test = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
walkdir = { workspace = true }
|
||||
http = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
temp-env = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -482,7 +482,7 @@ impl HealChannelProcessor {
|
||||
request_id: client_token,
|
||||
success: false,
|
||||
data: None,
|
||||
error: Some(error_text),
|
||||
error: Some(error_text.clone()),
|
||||
};
|
||||
let _ = response_tx.send(Ok(response.clone()));
|
||||
self.publish_response(response);
|
||||
|
||||
@@ -576,7 +576,7 @@ mod tests {
|
||||
assert_eq!(handler.event_count(), 1);
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
handler.add_event(event.clone());
|
||||
assert_eq!(handler.event_count(), 3);
|
||||
}
|
||||
|
||||
@@ -593,7 +593,7 @@ mod tests {
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event); // Should remove oldest
|
||||
handler.add_event(event.clone()); // Should remove oldest
|
||||
|
||||
assert_eq!(handler.event_count(), 2);
|
||||
}
|
||||
@@ -610,7 +610,7 @@ mod tests {
|
||||
};
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
handler.add_event(event.clone());
|
||||
|
||||
let events = handler.get_events();
|
||||
assert_eq!(events.len(), 2);
|
||||
|
||||
@@ -2892,7 +2892,7 @@ fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTa
|
||||
gauge!(
|
||||
"rustfs_heal_task_running",
|
||||
"type" => type_label.to_string(),
|
||||
"set" => set_label
|
||||
"set" => set_label.clone()
|
||||
)
|
||||
.set(count as f64);
|
||||
}
|
||||
@@ -3482,7 +3482,7 @@ mod tests {
|
||||
);
|
||||
|
||||
assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(runnable), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(runnable.clone()), QueuePushOutcome::Accepted);
|
||||
|
||||
let mut running = HashMap::new();
|
||||
running.insert("pool_0_set_1".to_string(), 1);
|
||||
|
||||
@@ -28,7 +28,6 @@ use rustfs_heal::heal::storage::{
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
future::Future,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
@@ -53,9 +52,15 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Disable the grace window while the destructive heal decision is evaluated.
|
||||
async fn with_dangling_grace_disabled<T>(future: impl Future<Output = T>) -> T {
|
||||
temp_env::async_with_vars([(GRACE_ENV, Some("0"))], future).await
|
||||
/// Disable the dangling-delete grace window so the destructive path is genuinely
|
||||
/// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD
|
||||
/// be dangling-deleted here. With grace at its 1h default the delete path would be
|
||||
/// masked and the test would prove nothing.
|
||||
fn disable_dangling_grace() {
|
||||
// Safe under nextest: each test runs in its own process and is `#[serial]`.
|
||||
unsafe {
|
||||
std::env::set_var(GRACE_ENV, "0");
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the
|
||||
@@ -245,6 +250,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn union_meta_lost_data_present_is_repaired_not_destroyed() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
|
||||
let bucket = "b920-meta-lost";
|
||||
let object = "obj.bin";
|
||||
@@ -262,10 +268,10 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
// Heal the version through the real heal storage (Deep).
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
assert!(
|
||||
error.is_none(),
|
||||
"recoverable version must heal without error (must NOT be dangling-deleted): {error:?}"
|
||||
@@ -294,6 +300,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-torn";
|
||||
let object = "obj.bin";
|
||||
@@ -311,10 +318,10 @@ mod serial_tests {
|
||||
// is a real destructive action, not a no-op).
|
||||
assert_eq!(count_part_files(&object_dir(&disk_paths[0], bucket, object)), 1);
|
||||
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
// A dangling delete reports the version as gone (FileVersionNotFound),
|
||||
// proving the destructive path fired for a genuinely torn write.
|
||||
assert!(error.is_some(), "a torn (< data_blocks) version must NOT be silently treated as healed");
|
||||
@@ -342,6 +349,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
|
||||
let bucket = "b920-reconstruct";
|
||||
let object = "obj.bin";
|
||||
@@ -365,10 +373,10 @@ mod serial_tests {
|
||||
"disk-walk must enumerate the reconstructable sub-quorum version"
|
||||
);
|
||||
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
assert!(error.is_none(), "reconstructable version must heal cleanly: {error:?}");
|
||||
|
||||
// part.* + xl.meta physically restored on the 4 wiped disks.
|
||||
@@ -455,6 +463,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn offline_disk_during_walk_does_not_dangling_delete() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-offline";
|
||||
let object = "obj.bin";
|
||||
@@ -482,7 +491,8 @@ mod serial_tests {
|
||||
remove: false,
|
||||
..Default::default()
|
||||
};
|
||||
let (_result, error) = with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &normal_opts))
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &normal_opts)
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
assert!(error.is_none(), "quorum-present object must not be destroyed: {error:?}");
|
||||
@@ -500,6 +510,7 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b1044-deep-verify";
|
||||
let object = "obj.bin";
|
||||
@@ -513,10 +524,10 @@ mod serial_tests {
|
||||
assert_eq!(count_part_files(&object_dir(disk, bucket, object)), 1, "intact shard must remain present");
|
||||
}
|
||||
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("deep heal_object call must not itself error");
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("deep heal_object call must not itself error");
|
||||
assert!(error.is_none(), "Deep heal must retain the three intact EC2+2 shards: {error:?}");
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -390,10 +390,8 @@ mod serial_tests {
|
||||
|
||||
// ─── 1️⃣ delete format.json on one disk ──────────────
|
||||
let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
|
||||
let failed_disk_path = disk_paths[0].with_extension("failed");
|
||||
std::fs::rename(&disk_paths[0], &failed_disk_path).expect("failed to detach disk_paths[0]");
|
||||
std::fs::remove_dir_all(&disk_paths[0]).expect("failed to delete all contents under disk_paths[0]");
|
||||
std::fs::create_dir_all(&disk_paths[0]).expect("failed to recreate disk_paths[0] directory");
|
||||
assert!(!format_path.exists(), "replacement disk already contains format.json before heal");
|
||||
println!("✅ Deleted format.json on disk: {:?}", disk_paths[0]);
|
||||
|
||||
let (_format_result, format_error) = heal_storage.heal_format(false).await.expect("failed to run heal_format");
|
||||
|
||||
@@ -25,9 +25,6 @@ keywords = ["rustfs", "openstack", "keystone", "authentication", "s3"]
|
||||
categories = ["authentication", "web-programming"]
|
||||
authors.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true, features = ["rt", "sync"] }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
|
||||
@@ -44,9 +44,7 @@ argon2 = { workspace = true }
|
||||
chacha20poly1305 = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
base64 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
subtle = { workspace = true }
|
||||
zeroize = { workspace = true, features = ["derive"] }
|
||||
|
||||
# Configuration and storage
|
||||
@@ -57,7 +55,7 @@ tempfile = { workspace = true }
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
|
||||
# Additional dependencies
|
||||
md-5 = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
arc-swap = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
|
||||
+126
-209
@@ -35,6 +35,7 @@ use std::collections::HashMap;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// Reject key identifiers that would not name a single file directly inside the key
|
||||
@@ -76,6 +77,8 @@ const LOCAL_KMS_ARGON2_P_COST: u32 = 1;
|
||||
/// Local KMS client that stores keys in local files
|
||||
pub struct LocalKmsClient {
|
||||
config: LocalConfig,
|
||||
/// In-memory cache of loaded keys for performance
|
||||
key_cache: RwLock<HashMap<String, MasterKeyInfo>>,
|
||||
/// Master encryption key for encrypting stored keys
|
||||
master_cipher: Option<Aes256Gcm>,
|
||||
/// Legacy pre-beta.9 master cipher for reading pre-Argon2 key files
|
||||
@@ -136,14 +139,13 @@ impl LocalKmsClient {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
let client = Self {
|
||||
Ok(Self {
|
||||
config,
|
||||
key_cache: RwLock::new(HashMap::new()),
|
||||
master_cipher,
|
||||
legacy_master_cipher,
|
||||
dek_crypto: AesDekCrypto::new(),
|
||||
};
|
||||
client.validate_existing_keys().await?;
|
||||
Ok(client)
|
||||
})
|
||||
}
|
||||
|
||||
/// Derive a 256-bit key from the master key string using a persistent Argon2id salt.
|
||||
@@ -191,35 +193,10 @@ impl LocalKmsClient {
|
||||
|
||||
let mut salt = [0u8; LOCAL_KMS_MASTER_KEY_SALT_LEN];
|
||||
rand::rng().fill(&mut salt[..]);
|
||||
let temp_path = config
|
||||
.key_dir
|
||||
.join(format!("{LOCAL_KMS_MASTER_KEY_SALT_FILE}.tmp-{}", uuid::Uuid::new_v4()));
|
||||
fs::write(&temp_path, salt).await?;
|
||||
Self::set_file_permissions(&temp_path, config.file_permissions).await?;
|
||||
match fs::hard_link(&temp_path, &salt_path).await {
|
||||
Ok(()) => {
|
||||
if let Err(error) = fs::remove_file(&temp_path).await {
|
||||
warn!(path = ?temp_path, %error, "Failed to remove Local KMS salt temporary file");
|
||||
}
|
||||
debug!(path = ?salt_path, "Local KMS master key salt created");
|
||||
Ok(salt)
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
let bytes = fs::read(&salt_path).await?;
|
||||
bytes.try_into().map_err(|_| {
|
||||
KmsError::configuration_error(format!(
|
||||
"Local KMS master key salt at {} must be exactly {} bytes",
|
||||
salt_path.display(),
|
||||
LOCAL_KMS_MASTER_KEY_SALT_LEN
|
||||
))
|
||||
})
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
Err(error.into())
|
||||
}
|
||||
}
|
||||
fs::write(&salt_path, salt).await?;
|
||||
Self::set_file_permissions(&salt_path, config.file_permissions).await?;
|
||||
debug!(path = ?salt_path, "Local KMS master key salt created");
|
||||
Ok(salt)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -265,12 +242,6 @@ impl LocalKmsClient {
|
||||
|
||||
let content = fs::read(&key_path).await?;
|
||||
let stored_key: StoredMasterKey = serde_json::from_slice(&content)?;
|
||||
if stored_key.key_id != key_id {
|
||||
return Err(KmsError::invalid_key(format!(
|
||||
"Local KMS key file identity mismatch: expected {key_id:?}, found {:?}",
|
||||
stored_key.key_id
|
||||
)));
|
||||
}
|
||||
|
||||
let encrypted_bytes = BASE64
|
||||
.decode(&stored_key.encrypted_key_material)
|
||||
@@ -355,43 +326,8 @@ impl LocalKmsClient {
|
||||
/// Save a master key to disk
|
||||
async fn save_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
|
||||
let key_path = self.master_key_path(&master_key.key_id)?;
|
||||
let content = self.encode_master_key(master_key, key_material)?;
|
||||
let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
|
||||
fs::write(&temp_path, &content).await?;
|
||||
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
|
||||
fs::rename(&temp_path, &key_path).await?;
|
||||
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn save_new_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
|
||||
let key_path = self.master_key_path(&master_key.key_id)?;
|
||||
let content = self.encode_master_key(master_key, key_material)?;
|
||||
let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
|
||||
fs::write(&temp_path, &content).await?;
|
||||
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
|
||||
|
||||
match fs::hard_link(&temp_path, &key_path).await {
|
||||
Ok(()) => {
|
||||
if let Err(error) = fs::remove_file(&temp_path).await {
|
||||
warn!(path = ?temp_path, %error, "Failed to remove Local KMS key temporary file");
|
||||
}
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key created");
|
||||
Ok(())
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
Err(KmsError::key_already_exists(&master_key.key_id))
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
Err(error.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<Vec<u8>> {
|
||||
// Encrypt key material if master cipher is available
|
||||
let (encrypted_key_material, nonce, at_rest_protection) = if let Some(ref cipher) = self.master_cipher {
|
||||
let mut nonce_bytes = [0u8; 12];
|
||||
rand::rng().fill(&mut nonce_bytes[..]);
|
||||
@@ -426,7 +362,17 @@ impl LocalKmsClient {
|
||||
at_rest_protection,
|
||||
};
|
||||
|
||||
serde_json::to_vec_pretty(&stored_key).map_err(Into::into)
|
||||
let content = serde_json::to_vec_pretty(&stored_key)?;
|
||||
|
||||
// Write to temporary file first, then rename for atomicity
|
||||
let temp_path = key_path.with_extension("tmp");
|
||||
fs::write(&temp_path, &content).await?;
|
||||
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
|
||||
|
||||
fs::rename(&temp_path, &key_path).await?;
|
||||
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the actual key material for a master key
|
||||
@@ -435,23 +381,6 @@ impl LocalKmsClient {
|
||||
Ok(key_material)
|
||||
}
|
||||
|
||||
async fn validate_existing_keys(&self) -> Result<()> {
|
||||
let mut entries = fs::read_dir(&self.config.key_dir).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
if path.extension().is_none_or(|extension| extension != "key") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let key_id = path
|
||||
.file_stem()
|
||||
.and_then(|stem| stem.to_str())
|
||||
.ok_or_else(|| KmsError::configuration_error("Local KMS key file name must be valid UTF-8"))?;
|
||||
self.decode_stored_key(key_id).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encrypt data using a master key
|
||||
async fn encrypt_with_master_key(&self, key_id: &str, plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
|
||||
// Load the actual master key material
|
||||
@@ -584,7 +513,11 @@ impl KmsClient for LocalKmsClient {
|
||||
let master_key = MasterKeyInfo::new_with_description(key_id.to_string(), algorithm.to_string(), Some(created_by), None);
|
||||
|
||||
// Save to disk
|
||||
self.save_new_master_key(&master_key, &key_material).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Cache the key
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
debug!(key_id, "Local KMS master key created");
|
||||
Ok(master_key)
|
||||
@@ -593,7 +526,23 @@ impl KmsClient for LocalKmsClient {
|
||||
async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> {
|
||||
debug!("Describing key: {}", key_id);
|
||||
|
||||
// Check cache first
|
||||
{
|
||||
let cache = self.key_cache.read().await;
|
||||
if let Some(master_key) = cache.get(key_id) {
|
||||
return Ok(master_key.clone().into());
|
||||
}
|
||||
}
|
||||
|
||||
// Load from disk
|
||||
let master_key = self.load_master_key(key_id).await?;
|
||||
|
||||
// Update cache
|
||||
{
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
}
|
||||
|
||||
Ok(master_key.into())
|
||||
}
|
||||
|
||||
@@ -653,6 +602,10 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key enabled");
|
||||
Ok(())
|
||||
}
|
||||
@@ -668,6 +621,10 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key disabled");
|
||||
Ok(())
|
||||
}
|
||||
@@ -689,6 +646,10 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key deletion scheduled");
|
||||
Ok(())
|
||||
}
|
||||
@@ -704,17 +665,31 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key deletion canceled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
|
||||
if !fs::try_exists(self.master_key_path(key_id)?).await? {
|
||||
return Err(KmsError::key_not_found(key_id));
|
||||
}
|
||||
Err(KmsError::invalid_operation(
|
||||
"Local KMS key rotation is unavailable until historical key versions can be retained",
|
||||
))
|
||||
debug!("Rotating key: {}", key_id);
|
||||
|
||||
let mut master_key = self.load_master_key(key_id).await?;
|
||||
master_key.version += 1;
|
||||
master_key.rotated_at = Some(Zoned::now());
|
||||
|
||||
// Generate new key material
|
||||
let key_material = generate_key_material(&master_key.algorithm)?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
debug!(key_id, "Local KMS key rotated");
|
||||
Ok(master_key)
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<()> {
|
||||
@@ -770,6 +745,11 @@ impl KmsBackend for LocalKmsBackend {
|
||||
async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
|
||||
let key_id = request.key_name.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
|
||||
|
||||
// `save_master_key` writes through a temp file and renames over the destination, so
|
||||
// creating a key under an existing name would replace its material and silently
|
||||
// destroy the ability to decrypt everything wrapped under it. The sibling
|
||||
// `KmsClient::create_key` has always refused this; the backend path did not, and
|
||||
// this is the path the admin API uses.
|
||||
if self.client.master_key_path(&key_id)?.exists() {
|
||||
return Err(KmsError::key_already_exists(&key_id));
|
||||
}
|
||||
@@ -787,8 +767,11 @@ impl KmsBackend for LocalKmsBackend {
|
||||
request.description.clone(),
|
||||
);
|
||||
|
||||
// Save to disk
|
||||
self.client.save_new_master_key(&master_key, &key_material).await?;
|
||||
// Save to disk and cache
|
||||
self.client.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.insert(key_id.clone(), master_key.clone());
|
||||
|
||||
master_key
|
||||
};
|
||||
@@ -905,6 +888,10 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.await
|
||||
.map_err(|e| KmsError::internal_error(format!("Failed to delete key file: {e}")))?;
|
||||
|
||||
// Remove from cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.remove(key_id);
|
||||
|
||||
debug!(key_id, "Local KMS key deleted immediately");
|
||||
|
||||
// Return success response for immediate deletion
|
||||
@@ -948,6 +935,10 @@ impl KmsBackend for LocalKmsBackend {
|
||||
|
||||
self.client.save_master_key(&master_key, &existing_key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
// Convert master_key to KeyMetadata for response
|
||||
let key_metadata = KeyMetadata {
|
||||
key_id: master_key.key_id.clone(),
|
||||
@@ -995,6 +986,10 @@ impl KmsBackend for LocalKmsBackend {
|
||||
|
||||
self.client.save_master_key(&master_key, &existing_key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
// Convert master_key to KeyMetadata for response
|
||||
let key_metadata = KeyMetadata {
|
||||
key_id: master_key.key_id.clone(),
|
||||
@@ -1194,18 +1189,6 @@ mod tests {
|
||||
.expect("stored encrypted key should deserialize");
|
||||
assert_eq!(stored.at_rest_protection, StoredKeyProtection::EncryptedMasterKey);
|
||||
assert_eq!(stored.nonce.len(), 12);
|
||||
|
||||
let wrong_master_error = match LocalKmsClient::new(LocalConfig {
|
||||
key_dir: client.config.key_dir.clone(),
|
||||
master_key: Some("wrong-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("wrong master key must fail initialization"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(wrong_master_error, KmsError::CryptographicError { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1245,57 +1228,17 @@ mod tests {
|
||||
master_key: None,
|
||||
file_permissions: Some(0o600),
|
||||
};
|
||||
let err = match LocalKmsClient::new(config).await {
|
||||
Ok(_) => panic!("initialization must reject an unreadable encrypted key"),
|
||||
Err(error) => error,
|
||||
};
|
||||
let client_without_master = LocalKmsClient::new(config)
|
||||
.await
|
||||
.expect("client without master key should still initialize in dev-mode tests");
|
||||
|
||||
let err = client_without_master
|
||||
.describe_key("encrypted-key", None)
|
||||
.await
|
||||
.expect_err("encrypted key should require a master key to read");
|
||||
assert!(err.to_string().contains("requires a configured master key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_key_rotation_is_rejected_without_overwriting_key_material() {
|
||||
let (client, _temp_dir) = create_test_client().await;
|
||||
let key_id = "rotation-key";
|
||||
client.create_key(key_id, "AES_256", None).await.expect("create key");
|
||||
let original_material = client.get_key_material(key_id).await.expect("load original material");
|
||||
|
||||
let error = client
|
||||
.rotate_key(key_id, None)
|
||||
.await
|
||||
.expect_err("rotation must remain unavailable without historical key versions");
|
||||
|
||||
assert!(matches!(error, KmsError::InvalidOperation { .. }));
|
||||
assert_eq!(
|
||||
client.get_key_material(key_id).await.expect("reload original material"),
|
||||
original_material
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn startup_rejects_key_file_with_mismatched_embedded_id() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("file-name", "AES_256", None).await.expect("create key");
|
||||
let key_path = client.master_key_path("file-name").expect("valid key id");
|
||||
let mut stored: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(&key_path).await.expect("read key file")).expect("decode key file");
|
||||
stored["key_id"] = serde_json::json!("embedded-name");
|
||||
fs::write(&key_path, serde_json::to_vec_pretty(&stored).expect("encode mismatched key"))
|
||||
.await
|
||||
.expect("write mismatched key");
|
||||
|
||||
let error = match LocalKmsClient::new(LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: None,
|
||||
file_permissions: Some(0o600),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("mismatched key identity must fail initialization"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(error, KmsError::InvalidKey { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_accepts_legacy_rfc3339_timestamp() {
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
@@ -1386,6 +1329,17 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("write beta.5 fixture");
|
||||
let mut explicit_protection = stored_key.clone();
|
||||
let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
|
||||
explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
|
||||
explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
|
||||
fs::write(
|
||||
temp_dir.path().join("beta5-explicit-key.key"),
|
||||
serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
|
||||
)
|
||||
.await
|
||||
.expect("write explicit-protection fixture");
|
||||
|
||||
let client = LocalKmsClient::new(LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: Some("beta5-test-master-key".to_string()),
|
||||
@@ -1399,34 +1353,24 @@ mod tests {
|
||||
.await
|
||||
.expect("decrypt beta.5 SHA-256 protected key");
|
||||
assert_eq!(material, vec![0x42; 32]);
|
||||
|
||||
let mut explicit_protection = stored_key.clone();
|
||||
let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
|
||||
explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
|
||||
explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
|
||||
fs::write(
|
||||
temp_dir.path().join("beta5-explicit-key.key"),
|
||||
serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
|
||||
)
|
||||
.await
|
||||
.expect("write explicit-protection fixture");
|
||||
let explicit_error = client
|
||||
.get_key_material("beta5-explicit-key")
|
||||
.await
|
||||
.expect_err("explicit current protection must not fall back to the beta.5 KDF");
|
||||
assert!(matches!(explicit_error, KmsError::CryptographicError { .. }));
|
||||
|
||||
let wrong_key_error = match LocalKmsClient::new(LocalConfig {
|
||||
let wrong_key_client = LocalKmsClient::new(LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: Some("wrong-beta5-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("wrong beta.5 master key must fail initialization"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(wrong_key_error, KmsError::CryptographicError { .. }));
|
||||
.expect("initialize local KMS with wrong beta.5 master key");
|
||||
let error = wrong_key_client
|
||||
.get_key_material("beta5-key")
|
||||
.await
|
||||
.expect_err("wrong beta.5 master key must not decrypt the fixture");
|
||||
assert!(matches!(error, KmsError::CryptographicError { .. }));
|
||||
}
|
||||
|
||||
/// R03-CAN-072 / R03-CAN-073: key identifiers arrive from request input, so every path
|
||||
@@ -1485,7 +1429,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// R07-CAN-103: creating a duplicate key must preserve its original material.
|
||||
/// R07-CAN-103: `save_master_key` writes a temp file and renames over the destination,
|
||||
/// so creating a key under an existing name would replace its material and silently
|
||||
/// destroy the ability to decrypt anything wrapped under it.
|
||||
#[tokio::test]
|
||||
async fn backend_create_key_refuses_to_replace_existing_key_material() {
|
||||
let temp_dir = TempDir::new().expect("Failed to create temp dir");
|
||||
@@ -1523,33 +1469,4 @@ mod tests {
|
||||
.expect("original key material must survive the refused create");
|
||||
assert_eq!(original, after, "existing key material must not be replaced");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_backend_create_allows_only_one_writer() {
|
||||
let temp_dir = TempDir::new().expect("create key directory");
|
||||
let config = || LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: Some("test-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
};
|
||||
let first = LocalKmsBackend {
|
||||
client: LocalKmsClient::new(config()).await.expect("create first client"),
|
||||
};
|
||||
let second = LocalKmsBackend {
|
||||
client: LocalKmsClient::new(config()).await.expect("create second client"),
|
||||
};
|
||||
let request = || CreateKeyRequest {
|
||||
key_name: Some("concurrent-key".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (first_result, second_result) = tokio::join!(first.create_key(request()), second.create_key(request()));
|
||||
assert_ne!(first_result.is_ok(), second_result.is_ok(), "exactly one create must succeed");
|
||||
let error = first_result
|
||||
.err()
|
||||
.or_else(|| second_result.err())
|
||||
.expect("one create must fail");
|
||||
assert!(matches!(error, KmsError::KeyAlreadyExists { .. }));
|
||||
assert_eq!(first.client.get_key_material("concurrent-key").await.expect("load key").len(), 32);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,8 +177,8 @@ mod tests {
|
||||
let service1 = manager.get_encryption_service().await.expect("Service should be available");
|
||||
|
||||
// Reconfigure to new service (zero-downtime)
|
||||
let mut config2 = config1;
|
||||
config2.timeout = std::time::Duration::from_secs(45);
|
||||
let temp_dir2 = TempDir::new().expect("Failed to create temp dir");
|
||||
let config2 = KmsConfig::local(temp_dir2.path().to_path_buf()).with_insecure_development_defaults();
|
||||
manager.reconfigure(config2).await.expect("Reconfiguration should succeed");
|
||||
|
||||
// Verify version 2
|
||||
|
||||
@@ -20,7 +20,6 @@ use crate::manager::KmsManager;
|
||||
use crate::types::*;
|
||||
use base64::Engine;
|
||||
use jiff::Zoned;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::random;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
@@ -28,12 +27,6 @@ use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tracing::debug;
|
||||
use zeroize::Zeroize;
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Data key for object encryption
|
||||
/// SECURITY: This struct automatically zeros sensitive key material when dropped
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -493,7 +486,8 @@ impl ObjectEncryptionService {
|
||||
|
||||
// Validate key MD5 if provided
|
||||
if let Some(expected_md5) = customer_key_md5 {
|
||||
let actual_md5_hex = md5_hex(customer_key);
|
||||
let actual_md5 = md5::compute(customer_key);
|
||||
let actual_md5_hex = format!("{actual_md5:x}");
|
||||
if actual_md5_hex != expected_md5.to_lowercase() {
|
||||
return Err(KmsError::validation_error("Customer key MD5 mismatch"));
|
||||
}
|
||||
|
||||
@@ -20,12 +20,10 @@ use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::service::ObjectEncryptionService;
|
||||
use arc_swap::ArcSwap;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use subtle::ConstantTimeEq;
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
@@ -33,53 +31,6 @@ const LOG_COMPONENT_KMS: &str = "kms";
|
||||
const LOG_SUBSYSTEM_SERVICE: &str = "service";
|
||||
const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
|
||||
|
||||
fn local_master_key_fingerprint(master_key: Option<&str>) -> [u8; 32] {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update([u8::from(master_key.is_some())]);
|
||||
if let Some(master_key) = master_key {
|
||||
digest.update(master_key.as_bytes());
|
||||
}
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
fn validate_local_transition(current: Option<&KmsConfig>, new: &KmsConfig) -> Result<()> {
|
||||
let Some(current) = current else {
|
||||
return Ok(());
|
||||
};
|
||||
let BackendConfig::Local(current_local) = ¤t.backend_config else {
|
||||
return Ok(());
|
||||
};
|
||||
let BackendConfig::Local(new_local) = &new.backend_config else {
|
||||
return Err(KmsError::configuration_error("Local KMS backend cannot be changed after configuration"));
|
||||
};
|
||||
|
||||
if current_local.key_dir != new_local.key_dir {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS key directory cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
if current_local.file_permissions != new_local.file_permissions {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS file permissions cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
if current.allow_insecure_dev_defaults != new.allow_insecure_dev_defaults {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS development mode cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
|
||||
let current_master_key = local_master_key_fingerprint(current_local.master_key.as_deref());
|
||||
let new_master_key = local_master_key_fingerprint(new_local.master_key.as_deref());
|
||||
if !bool::from(current_master_key.ct_eq(&new_master_key)) {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS master key cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// KMS service status
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub enum KmsServiceStatus {
|
||||
@@ -155,12 +106,7 @@ impl KmsServiceManager {
|
||||
|
||||
/// Configure KMS with new configuration
|
||||
pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
new_config.validate()?;
|
||||
{
|
||||
let config = self.config.read().await;
|
||||
validate_local_transition(config.as_ref(), &new_config)?;
|
||||
}
|
||||
|
||||
// Update configuration
|
||||
{
|
||||
@@ -308,9 +254,11 @@ impl KmsServiceManager {
|
||||
"KMS service reconfiguring"
|
||||
);
|
||||
new_config.validate()?;
|
||||
|
||||
// Configure with new config
|
||||
{
|
||||
let config = self.config.read().await;
|
||||
validate_local_transition(config.as_ref(), &new_config)?;
|
||||
let mut config = self.config.write().await;
|
||||
*config = Some(new_config.clone());
|
||||
}
|
||||
|
||||
// Create new service version without stopping old one
|
||||
@@ -320,11 +268,6 @@ impl KmsServiceManager {
|
||||
// Get old version for logging (lock-free read)
|
||||
let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version);
|
||||
|
||||
{
|
||||
let mut config = self.config.write().await;
|
||||
*config = Some(new_config);
|
||||
}
|
||||
|
||||
// Atomically switch to new service version (lock-free, instant CAS operation)
|
||||
// This is a true atomic operation - no waiting for locks, instant switch
|
||||
// Old service will be dropped when no more Arc references exist
|
||||
@@ -361,6 +304,8 @@ impl KmsServiceManager {
|
||||
Err(e) => {
|
||||
let err_msg = format!("Failed to reconfigure KMS: {e}");
|
||||
error!("{}", err_msg);
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Error(err_msg.clone());
|
||||
Err(KmsError::backend_error(&err_msg))
|
||||
}
|
||||
}
|
||||
@@ -532,105 +477,4 @@ mod tests {
|
||||
};
|
||||
assert!(static_config.secret_key.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forbidden_local_master_key_change_preserves_running_config_and_service() {
|
||||
use crate::types::{CreateKeyRequest, KeyUsage};
|
||||
use std::collections::HashMap;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let config = |master_key: &str| {
|
||||
let mut config = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
let BackendConfig::Local(local) = &mut config.backend_config else {
|
||||
panic!("local constructor must create local backend config");
|
||||
};
|
||||
local.master_key = Some(master_key.to_string());
|
||||
config.allow_insecure_dev_defaults = true;
|
||||
config
|
||||
};
|
||||
let manager = KmsServiceManager::new();
|
||||
manager
|
||||
.configure(config("working-master-key"))
|
||||
.await
|
||||
.expect("configure local KMS");
|
||||
manager.start().await.expect("start local KMS");
|
||||
manager
|
||||
.get_manager()
|
||||
.await
|
||||
.expect("running KMS manager")
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some("existing-key".to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
policy: None,
|
||||
tags: HashMap::new(),
|
||||
origin: None,
|
||||
})
|
||||
.await
|
||||
.expect("create encrypted key");
|
||||
let service_version = manager.get_service_version().await;
|
||||
|
||||
let error = manager
|
||||
.reconfigure(config("wrong-master-key"))
|
||||
.await
|
||||
.expect_err("local master key change must be rejected");
|
||||
|
||||
assert!(error.to_string().contains("master key cannot be changed"));
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
assert_eq!(manager.get_service_version().await, service_version);
|
||||
let current = manager.get_config().await.expect("working config must remain");
|
||||
let BackendConfig::Local(local) = current.backend_config else {
|
||||
panic!("working config must remain local");
|
||||
};
|
||||
assert_eq!(local.master_key.as_deref(), Some("working-master-key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configure_cannot_replace_existing_local_backend() {
|
||||
use base64::Engine as _;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let mut local = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
local.allow_insecure_dev_defaults = true;
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(local.clone()).await.expect("configure local KMS");
|
||||
|
||||
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
|
||||
let error = manager
|
||||
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
|
||||
.await
|
||||
.expect_err("existing local backend must be immutable");
|
||||
|
||||
assert!(error.to_string().contains("backend cannot be changed"));
|
||||
let current = manager.get_config().await.expect("local config must remain");
|
||||
assert!(matches!(current.backend_config, BackendConfig::Local(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconfigure_allows_safe_local_runtime_settings_only() {
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let mut initial = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
initial.allow_insecure_dev_defaults = true;
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(initial.clone()).await.expect("configure local KMS");
|
||||
manager.start().await.expect("start local KMS");
|
||||
|
||||
let mut updated = initial;
|
||||
updated.default_key_id = Some("evaluation-key".to_string());
|
||||
updated.timeout = std::time::Duration::from_secs(45);
|
||||
updated.enable_cache = false;
|
||||
manager
|
||||
.reconfigure(updated.clone())
|
||||
.await
|
||||
.expect("update safe local settings");
|
||||
|
||||
let current = manager.get_config().await.expect("updated config");
|
||||
assert_eq!(current.default_key_id, updated.default_key_id);
|
||||
assert_eq!(current.timeout, updated.timeout);
|
||||
assert!(!current.enable_cache);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -545,8 +545,6 @@ pub struct ScannerMetrics {
|
||||
pub collected_at: DateTime<Utc>,
|
||||
#[serde(rename = "current_cycle")]
|
||||
pub current_cycle: u64,
|
||||
#[serde(rename = "current_cycle_active", default, skip_serializing_if = "Option::is_none")]
|
||||
pub current_cycle_active: Option<bool>,
|
||||
#[serde(rename = "current_started")]
|
||||
pub current_started: DateTime<Utc>,
|
||||
#[serde(rename = "cycle_complete_times")]
|
||||
@@ -720,31 +718,7 @@ pub struct ScannerMetrics {
|
||||
}
|
||||
|
||||
impl ScannerMetrics {
|
||||
pub fn is_current_cycle_active(&self) -> bool {
|
||||
self.current_cycle_active.unwrap_or(self.current_cycle > 0)
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: &Self) {
|
||||
// Legacy nodes omit the activity field and use a non-zero cycle as
|
||||
// their active signal. New nodes publish explicit first-cycle and idle
|
||||
// states, including cycle zero.
|
||||
let self_cycle_active = self.is_current_cycle_active();
|
||||
let other_cycle_active = other.is_current_cycle_active();
|
||||
let self_cycle_authority = (
|
||||
self_cycle_active,
|
||||
self.current_cycle,
|
||||
self.cycles_completed_at.len(),
|
||||
self.cycles_completed_at.as_slice(),
|
||||
self.current_started,
|
||||
);
|
||||
let other_cycle_authority = (
|
||||
other_cycle_active,
|
||||
other.current_cycle,
|
||||
other.cycles_completed_at.len(),
|
||||
other.cycles_completed_at.as_slice(),
|
||||
other.current_started,
|
||||
);
|
||||
let other_cycle_is_authoritative = self_cycle_authority < other_cycle_authority;
|
||||
let other_is_newer = self.collected_at < other.collected_at;
|
||||
if other_is_newer {
|
||||
self.collected_at = other.collected_at;
|
||||
@@ -880,12 +854,15 @@ impl ScannerMetrics {
|
||||
self.ongoing_buckets = other.ongoing_buckets;
|
||||
}
|
||||
|
||||
if other_cycle_is_authoritative {
|
||||
if self.current_cycle < other.current_cycle {
|
||||
self.current_cycle = other.current_cycle;
|
||||
self.cycles_completed_at = other.cycles_completed_at.clone();
|
||||
self.current_started = other.current_started;
|
||||
}
|
||||
self.current_cycle_active = Some(self_cycle_active || other_cycle_active);
|
||||
|
||||
if other.cycles_completed_at.len() > self.cycles_completed_at.len() {
|
||||
self.cycles_completed_at = other.cycles_completed_at.clone();
|
||||
}
|
||||
|
||||
if !other.life_time_ops.is_empty() && self.life_time_ops.is_empty() {
|
||||
self.life_time_ops = other.life_time_ops.clone();
|
||||
@@ -954,13 +931,7 @@ impl Metrics {
|
||||
if let Some(scanner) = other.scanner.as_ref() {
|
||||
match self.scanner {
|
||||
Some(ref mut s_scanner) => s_scanner.merge(scanner),
|
||||
None => {
|
||||
let mut scanner = scanner.clone();
|
||||
if scanner.current_cycle_active.is_none() {
|
||||
scanner.current_cycle_active = Some(scanner.is_current_cycle_active());
|
||||
}
|
||||
self.scanner = Some(scanner);
|
||||
}
|
||||
None => self.scanner = Some(scanner.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1418,280 +1389,6 @@ pub struct Operations {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_serializes_cycle_active_presence() {
|
||||
let missing_value = serde_json::to_value(ScannerMetrics::default()).expect("scanner metrics should serialize");
|
||||
assert!(missing_value.get("current_cycle_active").is_none());
|
||||
let missing: ScannerMetrics =
|
||||
serde_json::from_value(missing_value).expect("older scanner metrics without cycle-active should decode");
|
||||
assert_eq!(missing.current_cycle_active, None);
|
||||
|
||||
let explicit_false_value = serde_json::to_value(ScannerMetrics {
|
||||
current_cycle_active: Some(false),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("scanner metrics with explicit cycle-active should serialize");
|
||||
assert_eq!(explicit_false_value["current_cycle_active"], serde_json::Value::Bool(false));
|
||||
let explicit_false: ScannerMetrics =
|
||||
serde_json::from_value(explicit_false_value).expect("explicit cycle-active should decode");
|
||||
assert_eq!(explicit_false.current_cycle_active, Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_prefers_an_active_first_cycle() {
|
||||
let collected_at = Utc::now();
|
||||
let idle_started = collected_at - chrono::Duration::hours(1);
|
||||
let active_started = collected_at - chrono::Duration::seconds(5);
|
||||
let mut scanner = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 0,
|
||||
current_cycle_active: Some(false),
|
||||
current_started: idle_started,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
scanner.merge(&ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle: 0,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: active_started,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_cycle, 0);
|
||||
assert_eq!(scanner.current_started, active_started);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metrics_merge_preserves_explicit_active_first_cycle() {
|
||||
let mut aggregated = Metrics::default();
|
||||
aggregated.merge(&Metrics {
|
||||
scanner: Some(ScannerMetrics {
|
||||
current_cycle_active: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let scanner = aggregated.scanner.expect("aggregated scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_cycle, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_preserves_legacy_nonzero_active_signal() {
|
||||
let collected_at = Utc::now();
|
||||
let mut scanner = ScannerMetrics {
|
||||
collected_at,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
scanner.merge(&ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle: 7,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_cycle, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metrics_merge_normalizes_first_legacy_scanner_snapshot() {
|
||||
let legacy = Metrics {
|
||||
scanner: Some(ScannerMetrics {
|
||||
current_cycle: 7,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let mut aggregated = Metrics::default();
|
||||
|
||||
aggregated.merge(&legacy);
|
||||
|
||||
let scanner = aggregated.scanner.expect("aggregated scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_cycle, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_preserves_explicit_inactive_nonzero_cycle() {
|
||||
let collected_at = Utc::now();
|
||||
let mut scanner = ScannerMetrics::default();
|
||||
|
||||
scanner.merge(&ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(false),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(false));
|
||||
assert_eq!(scanner.current_cycle, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_cycle_active_is_order_independent() {
|
||||
let collected_at = Utc::now();
|
||||
let legacy_active = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_started: collected_at - chrono::Duration::seconds(10),
|
||||
..Default::default()
|
||||
};
|
||||
let explicit_idle = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 0,
|
||||
current_cycle_active: Some(false),
|
||||
current_started: collected_at - chrono::Duration::hours(1),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut legacy_first = legacy_active.clone();
|
||||
legacy_first.merge(&explicit_idle);
|
||||
let mut legacy_second = explicit_idle.clone();
|
||||
legacy_second.merge(&legacy_active);
|
||||
assert_eq!(legacy_first.current_cycle_active, Some(true));
|
||||
assert_eq!(legacy_second.current_cycle_active, Some(true));
|
||||
assert_eq!(legacy_first.current_cycle, 7);
|
||||
assert_eq!(legacy_second.current_cycle, 7);
|
||||
|
||||
let explicit_inactive_nonzero = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(false),
|
||||
..Default::default()
|
||||
};
|
||||
let mut inactive_first = explicit_inactive_nonzero.clone();
|
||||
inactive_first.merge(&explicit_idle);
|
||||
let mut inactive_second = explicit_idle;
|
||||
inactive_second.merge(&explicit_inactive_nonzero);
|
||||
assert_eq!(inactive_first.current_cycle_active, Some(false));
|
||||
assert_eq!(inactive_second.current_cycle_active, Some(false));
|
||||
assert_eq!(inactive_first.current_cycle, 7);
|
||||
assert_eq!(inactive_second.current_cycle, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_cycle_authority_is_order_independent() {
|
||||
let collected_at = Utc::now();
|
||||
let completion = collected_at - chrono::Duration::minutes(1);
|
||||
let earlier_active = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: collected_at - chrono::Duration::seconds(10),
|
||||
cycles_completed_at: vec![completion],
|
||||
..Default::default()
|
||||
};
|
||||
let later_active = ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: collected_at - chrono::Duration::seconds(5),
|
||||
cycles_completed_at: vec![completion],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut earlier_first = earlier_active.clone();
|
||||
earlier_first.merge(&later_active);
|
||||
let mut later_first = later_active.clone();
|
||||
later_first.merge(&earlier_active);
|
||||
|
||||
assert_eq!(earlier_first.current_started, later_active.current_started);
|
||||
assert_eq!(later_first.current_started, later_active.current_started);
|
||||
assert_eq!(earlier_first.cycles_completed_at, later_active.cycles_completed_at);
|
||||
assert_eq!(later_first.cycles_completed_at, later_active.cycles_completed_at);
|
||||
|
||||
let stale_idle = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle_active: Some(false),
|
||||
current_started: collected_at - chrono::Duration::hours(1),
|
||||
..Default::default()
|
||||
};
|
||||
let completed_idle = ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle_active: Some(false),
|
||||
current_started: collected_at - chrono::Duration::seconds(5),
|
||||
cycles_completed_at: vec![completion],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut stale_first = stale_idle.clone();
|
||||
stale_first.merge(&completed_idle);
|
||||
let mut completed_first = completed_idle.clone();
|
||||
completed_first.merge(&stale_idle);
|
||||
|
||||
assert_eq!(stale_first.current_started, completed_idle.current_started);
|
||||
assert_eq!(completed_first.current_started, completed_idle.current_started);
|
||||
assert_eq!(stale_first.cycles_completed_at, completed_idle.cycles_completed_at);
|
||||
assert_eq!(completed_first.cycles_completed_at, completed_idle.cycles_completed_at);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_cycle_authority_is_associative() {
|
||||
let collected_at = Utc::now();
|
||||
let older_completion = collected_at - chrono::Duration::minutes(3);
|
||||
let last_completion = collected_at - chrono::Duration::minutes(1);
|
||||
let cycle_seven = ScannerMetrics {
|
||||
collected_at,
|
||||
current_cycle: 7,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: collected_at - chrono::Duration::seconds(10),
|
||||
cycles_completed_at: vec![older_completion, last_completion],
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_eight = ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::seconds(1),
|
||||
current_cycle: 8,
|
||||
current_cycle_active: Some(true),
|
||||
current_started: collected_at - chrono::Duration::seconds(5),
|
||||
cycles_completed_at: vec![last_completion],
|
||||
..Default::default()
|
||||
};
|
||||
let newer_idle = ScannerMetrics {
|
||||
collected_at: collected_at + chrono::Duration::hours(1),
|
||||
current_cycle_active: Some(false),
|
||||
current_started: collected_at - chrono::Duration::hours(1),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut left_associative = cycle_seven.clone();
|
||||
left_associative.merge(&cycle_eight);
|
||||
left_associative.merge(&newer_idle);
|
||||
|
||||
let mut right_group = cycle_eight.clone();
|
||||
right_group.merge(&newer_idle);
|
||||
let mut right_associative = cycle_seven.clone();
|
||||
right_associative.merge(&right_group);
|
||||
|
||||
assert_eq!(left_associative.current_cycle, 8);
|
||||
assert_eq!(right_associative.current_cycle, 8);
|
||||
assert_eq!(left_associative.current_started, cycle_eight.current_started);
|
||||
assert_eq!(right_associative.current_started, cycle_eight.current_started);
|
||||
assert_eq!(left_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
|
||||
assert_eq!(right_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
|
||||
|
||||
for order in [
|
||||
[&cycle_seven, &cycle_eight, &newer_idle],
|
||||
[&cycle_seven, &newer_idle, &cycle_eight],
|
||||
[&cycle_eight, &cycle_seven, &newer_idle],
|
||||
[&cycle_eight, &newer_idle, &cycle_seven],
|
||||
[&newer_idle, &cycle_seven, &cycle_eight],
|
||||
[&newer_idle, &cycle_eight, &cycle_seven],
|
||||
] {
|
||||
let mut merged = ScannerMetrics::default();
|
||||
for scanner in order {
|
||||
merged.merge(scanner);
|
||||
}
|
||||
assert_eq!(merged.current_cycle_active, Some(true));
|
||||
assert_eq!(merged.current_cycle, 8);
|
||||
assert_eq!(merged.current_started, cycle_eight.current_started);
|
||||
assert_eq!(merged.cycles_completed_at, cycle_eight.cycles_completed_at);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_merge_aggregates_partial_cycles_by_source() {
|
||||
let collected_at = Utc::now();
|
||||
|
||||
@@ -262,11 +262,11 @@ async fn obs_site_replication_stats() -> ReplicationStats {
|
||||
}
|
||||
|
||||
fn current_scanner_cycle_age_seconds(
|
||||
current_cycle_active: bool,
|
||||
current_cycle: u64,
|
||||
current_started: chrono::DateTime<Utc>,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> u64 {
|
||||
if !current_cycle_active {
|
||||
if current_cycle == 0 {
|
||||
0
|
||||
} else {
|
||||
now.signed_duration_since(current_started).num_seconds().max(0) as u64
|
||||
@@ -1090,7 +1090,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
let reference_time = metrics.cycles_completed_at.last().copied().unwrap_or(metrics.current_started);
|
||||
let last_activity_seconds = now.signed_duration_since(reference_time).num_seconds().max(0) as u64;
|
||||
let active_paths = metrics.active_scan_paths as u64;
|
||||
let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle_active, metrics.current_started, now);
|
||||
let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle, metrics.current_started, now);
|
||||
let current_scan_mode = scanner_scan_mode_code(&metrics.current_scan_mode);
|
||||
let current_cycle_age = current_cycle_age_seconds as f64;
|
||||
let last_cycle_duration = metrics.last_cycle_duration_seconds;
|
||||
@@ -1464,21 +1464,21 @@ mod tests {
|
||||
fn current_scanner_cycle_age_seconds_returns_zero_when_idle() {
|
||||
let now = Utc::now();
|
||||
|
||||
assert_eq!(current_scanner_cycle_age_seconds(false, now - chrono::Duration::seconds(30), now), 0);
|
||||
assert_eq!(current_scanner_cycle_age_seconds(0, now - chrono::Duration::seconds(30), now), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_scanner_cycle_age_seconds_clamps_future_start() {
|
||||
let now = Utc::now();
|
||||
|
||||
assert_eq!(current_scanner_cycle_age_seconds(true, now + chrono::Duration::seconds(30), now), 0);
|
||||
assert_eq!(current_scanner_cycle_age_seconds(4, now + chrono::Duration::seconds(30), now), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_scanner_cycle_age_seconds_reports_active_first_cycle_elapsed_time() {
|
||||
fn current_scanner_cycle_age_seconds_reports_active_elapsed_time() {
|
||||
let now = Utc::now();
|
||||
|
||||
assert_eq!(current_scanner_cycle_age_seconds(true, now - chrono::Duration::seconds(45), now), 45);
|
||||
assert_eq!(current_scanner_cycle_age_seconds(4, now - chrono::Duration::seconds(45), now), 45);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -25,9 +25,6 @@ description = "Protocol implementations for RustFS (FTPS, SFTP, etc.)"
|
||||
keywords = ["ftp", "sftp", "protocol", "storage", "rustfs"]
|
||||
categories = ["network-programming", "filesystem"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime", "dep:subtle"]
|
||||
@@ -47,7 +44,7 @@ swift = [
|
||||
"dep:tokio-util",
|
||||
"dep:serde",
|
||||
"dep:urlencoding",
|
||||
"dep:md-5",
|
||||
"dep:md5",
|
||||
"dep:quick-xml",
|
||||
"dep:hmac",
|
||||
"dep:sha1",
|
||||
@@ -111,7 +108,7 @@ http-body-util = { workspace = true, optional = true }
|
||||
tokio-util = { workspace = true, optional = true, features = ["rt", "io", "compat"] }
|
||||
serde = { workspace = true, optional = true, features = ["derive"] }
|
||||
urlencoding = { workspace = true, optional = true }
|
||||
md-5 = { workspace = true, optional = true }
|
||||
md5 = { workspace = true, optional = true }
|
||||
quick-xml = { workspace = true, optional = true, features = ["serialize"] }
|
||||
hmac = { workspace = true, optional = true }
|
||||
sha1 = { workspace = true, optional = true }
|
||||
@@ -137,7 +134,6 @@ socket2 = { workspace = true, optional = true, features = ["all"] }
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
proptest = "1"
|
||||
rustfs-test-utils = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "fs"] }
|
||||
|
||||
|
||||
@@ -608,7 +608,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.put_object_calls.push(PutObjectCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
metadata: input.metadata,
|
||||
metadata: input.metadata.clone(),
|
||||
});
|
||||
let stall = inner.stall_put_object;
|
||||
let entered = inner.put_object_entered.clone();
|
||||
@@ -731,7 +731,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.create_multipart_calls.push(CreateMultipartCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
metadata: input.metadata,
|
||||
metadata: input.metadata.clone(),
|
||||
});
|
||||
}
|
||||
match self.inner.lock().expect("lock").create_multipart_upload.pop_front() {
|
||||
@@ -749,7 +749,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.upload_part_calls.push(UploadPartCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
upload_id: input.upload_id,
|
||||
upload_id: input.upload_id.to_string(),
|
||||
part_number: input.part_number,
|
||||
content_length: input.content_length,
|
||||
});
|
||||
@@ -787,7 +787,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.complete_multipart_calls.push(CompleteCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
upload_id: input.upload_id,
|
||||
upload_id: input.upload_id.to_string(),
|
||||
part_count,
|
||||
});
|
||||
}
|
||||
@@ -808,7 +808,7 @@ impl StorageBackend for DummyBackend {
|
||||
inner.abort_multipart_calls.push(AbortCall {
|
||||
bucket: input.bucket.to_string(),
|
||||
key: input.key.to_string(),
|
||||
upload_id: input.upload_id,
|
||||
upload_id: input.upload_id.to_string(),
|
||||
});
|
||||
}
|
||||
match self.inner.lock().expect("lock").abort_multipart_upload.pop_front() {
|
||||
|
||||
@@ -379,7 +379,9 @@ where
|
||||
.await
|
||||
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
|
||||
|
||||
let prefix_with_slash = prefix.clone().map(|p| if p.ends_with('/') { p } else { format!("{}/", p) });
|
||||
let prefix_with_slash = prefix
|
||||
.clone()
|
||||
.map(|p| if p.ends_with('/') { p.to_string() } else { format!("{}/", p) });
|
||||
|
||||
let list_input = ListObjectsV2Input::builder()
|
||||
.bucket(bucket)
|
||||
|
||||
@@ -371,7 +371,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
|
||||
|
||||
let ftps_user = FtpsUser {
|
||||
username: principal.username.clone(),
|
||||
name: identity.credentials.name,
|
||||
name: identity.credentials.name.clone(),
|
||||
session_context,
|
||||
};
|
||||
|
||||
|
||||
@@ -1495,12 +1495,12 @@ mod tests {
|
||||
let buffer_len_u64 = part_buffer_len as u64;
|
||||
let phase = match phase_variant {
|
||||
0 => WritePhase::Buffering {
|
||||
part_buffer,
|
||||
part_buffer: part_buffer.clone(),
|
||||
},
|
||||
1 => WritePhase::Streaming {
|
||||
upload_id: "UP-proptest".to_string(),
|
||||
abort_authorized: true,
|
||||
part_buffer,
|
||||
part_buffer: part_buffer.clone(),
|
||||
uploaded_parts: Vec::new(),
|
||||
next_part_number,
|
||||
},
|
||||
|
||||
@@ -14,13 +14,14 @@
|
||||
|
||||
//! Swift account operations and validation
|
||||
|
||||
use super::metadata_update::{ACCOUNT_META_TAG_PREFIX, MetadataUpdate};
|
||||
use super::storage_api::account::{BucketOperations, MakeBucketOptions};
|
||||
use super::{SwiftError, SwiftResult};
|
||||
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, update_swift_bucket_tagging};
|
||||
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::dto::{Tag, Tagging};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use time;
|
||||
|
||||
/// Validate that the authenticated user has access to the requested account
|
||||
///
|
||||
@@ -131,8 +132,9 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
|
||||
if let Some(tagging) = &bucket_meta.tagging_config {
|
||||
for tag in &tagging.tag_set {
|
||||
if let (Some(key), Some(value)) = (&tag.key, &tag.value)
|
||||
&& let Some(meta_key) = key.strip_prefix(ACCOUNT_META_TAG_PREFIX)
|
||||
&& let Some(meta_key) = key.strip_prefix("swift-account-meta-")
|
||||
{
|
||||
// Strip "swift-account-meta-" prefix
|
||||
metadata.insert(meta_key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
@@ -146,46 +148,15 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
|
||||
/// Updates account-level metadata such as TempURL keys.
|
||||
/// Only updates swift-account-meta-* tags, preserving other tags.
|
||||
///
|
||||
/// Swift account POST is additive: `update` names the items to write and the
|
||||
/// items to drop, and everything else keeps its stored value. Replacing the
|
||||
/// whole set instead would make an unrelated POST — setting a quota, say —
|
||||
/// delete the account's TempURL signing key, permanently invalidating every
|
||||
/// outstanding TempURL and FormPost signature for the account.
|
||||
///
|
||||
/// The caller must own the account: this metadata holds the account's TempURL
|
||||
/// signing key, so writing it for someone else's account would let the writer
|
||||
/// mint valid pre-signed URLs against that account's objects. Reads
|
||||
/// ([`get_account_metadata`]) stay unauthenticated on purpose — TempURL
|
||||
/// signature validation has to resolve the key before any credentials exist.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `account` - Account identifier
|
||||
/// * `update` - Metadata items to set and to remove (names are prefixed with `swift-account-meta-`)
|
||||
/// * `credentials` - Keystone credentials of the caller
|
||||
/// * `metadata` - Metadata key-value pairs to store (keys will be prefixed with `swift-account-meta-`)
|
||||
/// * `credentials` - S3 credentials
|
||||
pub async fn update_account_metadata(
|
||||
account: &str,
|
||||
update: &MetadataUpdate,
|
||||
credentials: &Option<Credentials>,
|
||||
metadata: &HashMap<String, String>,
|
||||
_credentials: &Option<Credentials>,
|
||||
) -> SwiftResult<()> {
|
||||
let Some(credentials) = credentials.as_ref() else {
|
||||
return Err(SwiftError::Unauthorized(
|
||||
"Keystone authentication required to update account metadata".to_string(),
|
||||
));
|
||||
};
|
||||
validate_account_access(account, credentials)?;
|
||||
|
||||
// These tags are persisted into the bucket metadata file, which every
|
||||
// later config write rewrites in full — so unbounded metadata inflates
|
||||
// the cost of unrelated writes for the life of the account. The item
|
||||
// count is capped against the merged result, inside the rewrite.
|
||||
update.validate()?;
|
||||
|
||||
// An update that names no item changes nothing, so there is no reason to
|
||||
// bring the account's metadata bucket into existence for it.
|
||||
if update.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let bucket_name = get_account_metadata_bucket_name(account);
|
||||
|
||||
let Some(store) = resolve_swift_object_store_handle() else {
|
||||
@@ -202,10 +173,59 @@ pub async fn update_account_metadata(
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to create account metadata bucket: {}", e)))?;
|
||||
}
|
||||
|
||||
// Merge into the persisted tags: only the swift-account-meta-* items this
|
||||
// update names change, and non-Swift tags are left alone. An empty result
|
||||
// clears the tagging config.
|
||||
update_swift_bucket_tagging(bucket_name, |current| update.apply_to_tags(current, ACCOUNT_META_TAG_PREFIX)).await
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
|
||||
// Get existing tags, preserving non-Swift tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
// Remove old swift-account-meta-* tags while preserving other tags
|
||||
existing_tagging.tag_set.retain(|tag| {
|
||||
if let Some(key) = &tag.key {
|
||||
!key.starts_with("swift-account-meta-")
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
// Add new metadata tags
|
||||
for (key, value) in metadata {
|
||||
existing_tagging.tag_set.push(Tag {
|
||||
key: Some(format!("swift-account-meta-{}", key)),
|
||||
value: Some(value.clone()),
|
||||
});
|
||||
}
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
if existing_tagging.tag_set.is_empty() {
|
||||
// No tags remain; clear tagging config
|
||||
bucket_meta_clone.tagging_config_xml = Vec::new();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = None;
|
||||
} else {
|
||||
// Serialize tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
}
|
||||
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name.clone(), bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get TempURL key for account
|
||||
|
||||
@@ -627,7 +627,9 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_paths_with_empty_lines() {
|
||||
let body = "/container1/file1.txt\n\n/container2/file2.txt\n \n/container1/file3.txt";
|
||||
assert_eq!(body.lines().filter(|line| !line.trim().is_empty()).count(), 3);
|
||||
let paths: Vec<&str> = body.lines().filter(|line| !line.trim().is_empty()).collect();
|
||||
|
||||
assert_eq!(paths.len(), 3);
|
||||
}
|
||||
|
||||
/// Tests for the `extract_tar_entries` async function.
|
||||
|
||||
@@ -17,13 +17,12 @@
|
||||
//! This module implements Swift container CRUD operations and container-bucket translation.
|
||||
|
||||
use super::account::validate_account_access;
|
||||
use super::metadata_update::{CONTAINER_META_TAG_PREFIX, MetadataUpdate};
|
||||
use super::storage_api::container::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations as _, MakeBucketOptions,
|
||||
};
|
||||
use super::types::Container;
|
||||
use super::{SwiftError, SwiftResult};
|
||||
use super::{get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging};
|
||||
use super::{get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, set_swift_bucket_metadata};
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::dto::{Tag, Tagging};
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -60,6 +59,30 @@ fn sanitize_storage_error<E: std::fmt::Display>(operation: &str, error: E) -> Sw
|
||||
SwiftError::InternalServerError(format!("{} operation failed", operation))
|
||||
}
|
||||
|
||||
/// Convert Swift container metadata to S3 tags
|
||||
///
|
||||
/// Swift container metadata uses X-Container-Meta-* headers.
|
||||
/// We store these as S3 tags with "swift-meta-" prefix to distinguish from regular bucket tags.
|
||||
///
|
||||
/// Example: X-Container-Meta-Color: Blue → S3 Tag: swift-meta-color=Blue
|
||||
fn swift_metadata_to_s3_tags(metadata: &std::collections::HashMap<String, String>) -> Option<Tagging> {
|
||||
let mut tags = Vec::new();
|
||||
|
||||
for (key, value) in metadata {
|
||||
// Store with "swift-meta-" prefix to namespace container metadata
|
||||
tags.push(Tag {
|
||||
key: Some(format!("swift-meta-{}", key.to_lowercase())),
|
||||
value: Some(value.clone()),
|
||||
});
|
||||
}
|
||||
|
||||
if tags.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Tagging { tag_set: tags })
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert S3 tags back to Swift container metadata
|
||||
///
|
||||
/// Extracts only tags with "swift-meta-" prefix, which represent Swift container metadata.
|
||||
@@ -68,9 +91,11 @@ fn s3_tags_to_swift_metadata(tagging: &Tagging) -> std::collections::HashMap<Str
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
|
||||
for tag in &tagging.tag_set {
|
||||
// Only process tags with "swift-meta-" prefix
|
||||
if let (Some(key), Some(value)) = (&tag.key, &tag.value)
|
||||
&& let Some(meta_key) = key.strip_prefix(CONTAINER_META_TAG_PREFIX)
|
||||
&& let Some(meta_key) = key.strip_prefix("swift-meta-")
|
||||
{
|
||||
// Skip "swift-meta-"
|
||||
metadata.insert(meta_key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
@@ -445,15 +470,12 @@ pub async fn get_container_metadata(account: &str, container: &str, credentials:
|
||||
/// - Returns 204 No Content on success
|
||||
/// - Returns 404 Not Found if container doesn't exist
|
||||
/// - Metadata is provided via X-Container-Meta-* headers
|
||||
/// - The update is additive: items the request does not name keep their stored
|
||||
/// value, and removal is explicit, via `X-Remove-Container-Meta-{name}` or an
|
||||
/// empty value
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn update_container_metadata(
|
||||
account: &str,
|
||||
container: &str,
|
||||
credentials: &Credentials,
|
||||
update: MetadataUpdate,
|
||||
metadata: std::collections::HashMap<String, String>,
|
||||
) -> SwiftResult<()> {
|
||||
// Validate account access and extract project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -461,12 +483,6 @@ pub async fn update_container_metadata(
|
||||
// Validate container name
|
||||
validate_container_name(container)?;
|
||||
|
||||
// These tags are persisted into the bucket metadata file, which every
|
||||
// later config write rewrites in full — so unbounded metadata inflates
|
||||
// the cost of unrelated writes for the life of the container. The item
|
||||
// count is capped against the merged result, inside the rewrite.
|
||||
update.validate()?;
|
||||
|
||||
// Create mapper with default config (tenant prefixing enabled)
|
||||
let mapper = ContainerMapper::default();
|
||||
|
||||
@@ -478,9 +494,7 @@ pub async fn update_container_metadata(
|
||||
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
|
||||
};
|
||||
|
||||
// Verify container exists. Checked before the empty-update shortcut below,
|
||||
// so a POST to a container that does not exist still answers 404 whether
|
||||
// or not it carried metadata.
|
||||
// Verify container exists
|
||||
store
|
||||
.get_bucket_info(&bucket_name, &BucketOptions::default())
|
||||
.await
|
||||
@@ -492,19 +506,59 @@ pub async fn update_container_metadata(
|
||||
}
|
||||
})?;
|
||||
|
||||
// An update that names no item leaves stored metadata alone, so skip the
|
||||
// persisted write and the peer reload it triggers rather than rewriting
|
||||
// the config to its current value. The handler reaches here on every
|
||||
// container POST, including ACL-only and versioning-only ones.
|
||||
if update.is_empty() {
|
||||
return Ok(());
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
|
||||
// Get existing tags, preserving non-Swift tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
// Remove old swift-meta-* tags while preserving other tags
|
||||
existing_tagging.tag_set.retain(|tag| {
|
||||
if let Some(key) = &tag.key {
|
||||
!key.starts_with("swift-meta-")
|
||||
} else {
|
||||
true // Keep tags with no key (shouldn't happen, but be safe)
|
||||
}
|
||||
});
|
||||
|
||||
// Add new Swift metadata tags if provided
|
||||
if let Some(mut new_tagging) = swift_metadata_to_s3_tags(&metadata) {
|
||||
// Merge: existing non-Swift tags + new Swift tags
|
||||
existing_tagging.tag_set.append(&mut new_tagging.tag_set);
|
||||
}
|
||||
// If metadata.is_empty() and swift_metadata_to_s3_tags returns None,
|
||||
// we've already removed swift-meta-* tags above, so only non-Swift tags remain
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
if existing_tagging.tag_set.is_empty() {
|
||||
// No tags remain after removing swift-meta-* tags; clear tagging config
|
||||
bucket_meta_clone.tagging_config_xml = Vec::new();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = None;
|
||||
} else {
|
||||
// Serialize the merged tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
}
|
||||
|
||||
// Merge into the persisted tags: only the swift-meta-* items this update
|
||||
// names change, so the container's other metadata — and the ACL and
|
||||
// versioning tags sharing this tag set — survive. An empty result clears
|
||||
// the tagging config.
|
||||
update_swift_bucket_tagging(bucket_name, |current| update.apply_to_tags(current, CONTAINER_META_TAG_PREFIX)).await
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete a container
|
||||
@@ -765,23 +819,44 @@ pub async fn enable_versioning(
|
||||
}
|
||||
})?;
|
||||
|
||||
// Rewrite the persisted tags: replace any versioning tag with the new
|
||||
// archive location while preserving all other tags.
|
||||
update_swift_bucket_tagging(bucket_name, |current| {
|
||||
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
|
||||
tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some("swift-versions-location".to_string()),
|
||||
value: Some(archive_container.to_string()), // Store Swift container name, not S3 bucket name
|
||||
});
|
||||
// Get existing tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
Ok(tagging)
|
||||
})
|
||||
.await?;
|
||||
// Remove old versioning tag if present
|
||||
existing_tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
|
||||
|
||||
// Add new versioning tag
|
||||
existing_tagging.tag_set.push(Tag {
|
||||
key: Some("swift-versions-location".to_string()),
|
||||
value: Some(archive_container.to_string()), // Store Swift container name, not S3 bucket name
|
||||
});
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
// Serialize tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -807,38 +882,50 @@ pub async fn disable_versioning(account: &str, container: &str, credentials: &Cr
|
||||
let mapper = ContainerMapper::default();
|
||||
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
|
||||
|
||||
let Some(store) = resolve_swift_object_store_handle() else {
|
||||
// Verify container exists
|
||||
let Some(_store) = resolve_swift_object_store_handle() else {
|
||||
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
|
||||
};
|
||||
|
||||
// Verify container exists. Without this the rewrite below would persist a
|
||||
// fabricated default for a container that does not exist: the metadata
|
||||
// loader turns "no metadata on disk" into a fresh BucketMetadata, and
|
||||
// writing that creates an orphan .metadata.bin and caches a fabricated
|
||||
// default as authoritative.
|
||||
store
|
||||
.get_bucket_info(&bucket_name, &BucketOptions::default())
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.to_string().contains("not found") || e.to_string().contains("NoSuchBucket") {
|
||||
SwiftError::NotFound(format!("Container '{}' not found", container))
|
||||
} else {
|
||||
sanitize_storage_error("Container verification", e)
|
||||
}
|
||||
})?;
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
|
||||
// Rewrite the persisted tags: drop the versioning tag while preserving
|
||||
// all other tags. An empty result clears the tagging config.
|
||||
update_swift_bucket_tagging(bucket_name, |current| {
|
||||
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
|
||||
tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
|
||||
// Get existing tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
Ok(tagging)
|
||||
})
|
||||
.await?;
|
||||
// Remove versioning tag
|
||||
existing_tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
if existing_tagging.tag_set.is_empty() {
|
||||
// No tags remain; clear tagging config
|
||||
bucket_meta_clone.tagging_config_xml = Vec::new();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = None;
|
||||
} else {
|
||||
// Serialize remaining tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
}
|
||||
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -963,37 +1050,65 @@ pub async fn set_container_acl(
|
||||
}
|
||||
})?;
|
||||
|
||||
// Rewrite the persisted tags: replace the ACL tags with the new grants
|
||||
// while preserving all other tags. An empty result clears the tagging
|
||||
// config.
|
||||
update_swift_bucket_tagging(bucket_name, |current| {
|
||||
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
|
||||
tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-acl-read") && tag.key.as_deref() != Some("swift-acl-write"));
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
|
||||
if let Some(read) = read_acl
|
||||
&& !read.trim().is_empty()
|
||||
{
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some("swift-acl-read".to_string()),
|
||||
value: Some(read.to_string()),
|
||||
});
|
||||
}
|
||||
// Get existing tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
if let Some(write) = write_acl
|
||||
&& !write.trim().is_empty()
|
||||
{
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some("swift-acl-write".to_string()),
|
||||
value: Some(write.to_string()),
|
||||
});
|
||||
}
|
||||
// Remove old ACL tags
|
||||
existing_tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-acl-read") && tag.key.as_deref() != Some("swift-acl-write"));
|
||||
|
||||
Ok(tagging)
|
||||
})
|
||||
.await?;
|
||||
// Add new read ACL tag if provided
|
||||
if let Some(read) = read_acl
|
||||
&& !read.trim().is_empty()
|
||||
{
|
||||
existing_tagging.tag_set.push(Tag {
|
||||
key: Some("swift-acl-read".to_string()),
|
||||
value: Some(read.to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
// Add new write ACL tag if provided
|
||||
if let Some(write) = write_acl
|
||||
&& !write.trim().is_empty()
|
||||
{
|
||||
existing_tagging.tag_set.push(Tag {
|
||||
key: Some("swift-acl-write".to_string()),
|
||||
value: Some(write.to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
if existing_tagging.tag_set.is_empty() {
|
||||
// No tags remain; clear tagging config
|
||||
bucket_meta_clone.tagging_config_xml = Vec::new();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = None;
|
||||
} else {
|
||||
// Serialize tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
}
|
||||
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
|
||||
debug!(
|
||||
"Set ACLs for container {}/{}: read={:?}, write={:?}",
|
||||
@@ -1339,6 +1454,64 @@ mod tests {
|
||||
assert!(first_char.is_ascii_lowercase() || first_char.is_ascii_digit());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_swift_metadata_to_s3_tags() {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("color".to_string(), "blue".to_string());
|
||||
metadata.insert("description".to_string(), "test container".to_string());
|
||||
|
||||
let tagging = swift_metadata_to_s3_tags(&metadata).unwrap();
|
||||
assert_eq!(tagging.tag_set.len(), 2);
|
||||
|
||||
// Verify tags have swift-meta- prefix
|
||||
let color_tag = tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.find(|t| t.key.as_deref() == Some("swift-meta-color"))
|
||||
.expect("color tag not found");
|
||||
assert_eq!(color_tag.value.as_deref(), Some("blue"));
|
||||
|
||||
let desc_tag = tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.find(|t| t.key.as_deref() == Some("swift-meta-description"))
|
||||
.expect("description tag not found");
|
||||
assert_eq!(desc_tag.value.as_deref(), Some("test container"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_swift_metadata_to_s3_tags_empty() {
|
||||
let metadata = std::collections::HashMap::new();
|
||||
let tagging = swift_metadata_to_s3_tags(&metadata);
|
||||
assert!(tagging.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_swift_metadata_to_s3_tags_case_normalization() {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
metadata.insert("Color".to_string(), "Red".to_string());
|
||||
metadata.insert("PRIORITY".to_string(), "High".to_string());
|
||||
|
||||
let tagging = swift_metadata_to_s3_tags(&metadata).unwrap();
|
||||
|
||||
// Keys should be lowercased
|
||||
assert!(tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("swift-meta-color")));
|
||||
assert!(
|
||||
tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.any(|t| t.key.as_deref() == Some("swift-meta-priority"))
|
||||
);
|
||||
|
||||
// Values should be preserved as-is
|
||||
let color_tag = tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.find(|t| t.key.as_deref() == Some("swift-meta-color"))
|
||||
.unwrap();
|
||||
assert_eq!(color_tag.value.as_deref(), Some("Red"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_s3_tags_to_swift_metadata() {
|
||||
let tagging = Tagging {
|
||||
@@ -1394,80 +1567,91 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_metadata_roundtrip() {
|
||||
// What a POST writes must be what a HEAD reads back: run the items
|
||||
// through the tag-merge write path and the tag-parse read path.
|
||||
let update = MetadataUpdate::default()
|
||||
.set("color", "blue")
|
||||
.set("owner", "alice")
|
||||
.set("priority", "high");
|
||||
// Test that we can convert metadata -> tags -> metadata without loss
|
||||
let mut original_metadata = std::collections::HashMap::new();
|
||||
original_metadata.insert("color".to_string(), "blue".to_string());
|
||||
original_metadata.insert("owner".to_string(), "alice".to_string());
|
||||
original_metadata.insert("priority".to_string(), "high".to_string());
|
||||
|
||||
let tagging = update
|
||||
.apply_to_tags(None, CONTAINER_META_TAG_PREFIX)
|
||||
.expect("merge should be accepted");
|
||||
let recovered = s3_tags_to_swift_metadata(&tagging);
|
||||
let tagging = swift_metadata_to_s3_tags(&original_metadata).unwrap();
|
||||
let recovered_metadata = s3_tags_to_swift_metadata(&tagging);
|
||||
|
||||
assert_eq!(recovered.len(), 3);
|
||||
assert_eq!(recovered.get("color").map(String::as_str), Some("blue"));
|
||||
assert_eq!(recovered.get("owner").map(String::as_str), Some("alice"));
|
||||
assert_eq!(recovered.get("priority").map(String::as_str), Some("high"));
|
||||
assert_eq!(recovered_metadata.len(), original_metadata.len());
|
||||
for (key, value) in &original_metadata {
|
||||
assert_eq!(
|
||||
recovered_metadata.get(&key.to_lowercase()),
|
||||
Some(value),
|
||||
"Metadata key {} not preserved in roundtrip",
|
||||
key
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tag_preservation_merge_with_existing() {
|
||||
// A container metadata POST shares its tag set with the container ACL
|
||||
// and versioning tags, and with whatever S3 tags the bucket carries.
|
||||
// Only the swift-meta-* items the POST names may change.
|
||||
let existing = Tagging {
|
||||
// Test merging Swift metadata with existing non-Swift tags
|
||||
let mut existing_tagging = Tagging {
|
||||
tag_set: vec![
|
||||
Tag {
|
||||
key: Some("swift-meta-color".to_string()),
|
||||
value: Some("blue".to_string()),
|
||||
},
|
||||
Tag {
|
||||
key: Some("swift-acl-read".to_string()),
|
||||
value: Some(".r:*".to_string()),
|
||||
},
|
||||
Tag {
|
||||
key: Some("swift-versions-location".to_string()),
|
||||
value: Some("archive".to_string()),
|
||||
},
|
||||
Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("production".to_string()),
|
||||
},
|
||||
Tag {
|
||||
key: Some("team".to_string()),
|
||||
value: Some("backend".to_string()),
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
let merged = MetadataUpdate::default()
|
||||
.set("description", "test")
|
||||
.apply_to_tags(Some(&existing), CONTAINER_META_TAG_PREFIX)
|
||||
.expect("merge should be accepted");
|
||||
let tag = |key: &str| {
|
||||
merged
|
||||
.tag_set
|
||||
.iter()
|
||||
.find(|t| t.key.as_deref() == Some(key))
|
||||
.and_then(|t| t.value.as_deref())
|
||||
};
|
||||
// Remove old swift-meta-* tags
|
||||
existing_tagging.tag_set.retain(|tag| {
|
||||
if let Some(key) = &tag.key {
|
||||
!key.starts_with("swift-meta-")
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
assert_eq!(tag("swift-meta-description"), Some("test"), "the new item should be added");
|
||||
assert_eq!(tag("swift-meta-color"), Some("blue"), "an unnamed item should be preserved");
|
||||
assert_eq!(tag("swift-acl-read"), Some(".r:*"), "the ACL tag should be preserved");
|
||||
assert_eq!(tag("swift-versions-location"), Some("archive"), "the versioning tag should be preserved");
|
||||
assert_eq!(tag("env"), Some("production"), "non-Swift tags should be preserved");
|
||||
assert_eq!(merged.tag_set.len(), 5);
|
||||
// Add new Swift metadata
|
||||
let mut new_metadata = std::collections::HashMap::new();
|
||||
new_metadata.insert("description".to_string(), "test".to_string());
|
||||
let mut new_tagging = swift_metadata_to_s3_tags(&new_metadata).unwrap();
|
||||
|
||||
// Merge
|
||||
existing_tagging.tag_set.append(&mut new_tagging.tag_set);
|
||||
|
||||
// Verify: should have env, team, and new swift-meta-description
|
||||
assert_eq!(existing_tagging.tag_set.len(), 3);
|
||||
|
||||
let has_env = existing_tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("env"));
|
||||
let has_team = existing_tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("team"));
|
||||
let has_description = existing_tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.any(|t| t.key.as_deref() == Some("swift-meta-description"));
|
||||
|
||||
assert!(has_env, "env tag should be preserved");
|
||||
assert!(has_team, "team tag should be preserved");
|
||||
assert!(has_description, "swift-meta-description should be added");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tag_preservation_remove_only_swift() {
|
||||
// Removing the last container metadata item leaves the other tags —
|
||||
// and so must not clear the tagging config.
|
||||
let existing = Tagging {
|
||||
// Test that clearing Swift metadata preserves non-Swift tags
|
||||
let mut existing_tagging = Tagging {
|
||||
tag_set: vec![
|
||||
Tag {
|
||||
key: Some("swift-meta-color".to_string()),
|
||||
value: Some("blue".to_string()),
|
||||
},
|
||||
Tag {
|
||||
key: Some("swift-meta-owner".to_string()),
|
||||
value: Some("alice".to_string()),
|
||||
},
|
||||
Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("production".to_string()),
|
||||
@@ -1479,28 +1663,37 @@ mod tests {
|
||||
],
|
||||
};
|
||||
|
||||
let merged = MetadataUpdate::default()
|
||||
.remove("color")
|
||||
.apply_to_tags(Some(&existing), CONTAINER_META_TAG_PREFIX)
|
||||
.expect("merge should be accepted");
|
||||
// Remove swift-meta-* tags (simulating empty metadata update)
|
||||
existing_tagging.tag_set.retain(|tag| {
|
||||
if let Some(key) = &tag.key {
|
||||
!key.starts_with("swift-meta-")
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
assert_eq!(merged.tag_set.len(), 2);
|
||||
assert!(merged.tag_set.iter().any(|t| t.key.as_deref() == Some("env")));
|
||||
assert!(merged.tag_set.iter().any(|t| t.key.as_deref() == Some("cost-center")));
|
||||
assert!(
|
||||
!merged
|
||||
.tag_set
|
||||
.iter()
|
||||
.any(|t| t.key.as_ref().is_some_and(|k| k.starts_with(CONTAINER_META_TAG_PREFIX))),
|
||||
"the removed item should be gone"
|
||||
);
|
||||
// Verify: should only have env and cost-center
|
||||
assert_eq!(existing_tagging.tag_set.len(), 2);
|
||||
|
||||
let has_env = existing_tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("env"));
|
||||
let has_cost_center = existing_tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.any(|t| t.key.as_deref() == Some("cost-center"));
|
||||
let has_swift_meta = existing_tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.any(|t| t.key.as_ref().is_some_and(|k| k.starts_with("swift-meta-")));
|
||||
|
||||
assert!(has_env, "env tag should be preserved");
|
||||
assert!(has_cost_center, "cost-center tag should be preserved");
|
||||
assert!(!has_swift_meta, "all swift-meta-* tags should be removed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tag_preservation_empty_after_swift_removal() {
|
||||
// Removing every item when nothing else is tagged empties the tag set,
|
||||
// which is how the caller knows to clear the tagging config.
|
||||
let existing = Tagging {
|
||||
// Test that if only Swift tags exist, clearing them results in empty tagging
|
||||
let mut existing_tagging = Tagging {
|
||||
tag_set: vec![
|
||||
Tag {
|
||||
key: Some("swift-meta-color".to_string()),
|
||||
@@ -1513,13 +1706,38 @@ mod tests {
|
||||
],
|
||||
};
|
||||
|
||||
let merged = MetadataUpdate::default()
|
||||
.remove("color")
|
||||
.remove("owner")
|
||||
.apply_to_tags(Some(&existing), CONTAINER_META_TAG_PREFIX)
|
||||
.expect("merge should be accepted");
|
||||
// Remove swift-meta-* tags
|
||||
existing_tagging.tag_set.retain(|tag| {
|
||||
if let Some(key) = &tag.key {
|
||||
!key.starts_with("swift-meta-")
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
assert!(merged.tag_set.is_empty(), "tagging should be empty after removing every swift-meta-* tag");
|
||||
// Verify: should be empty
|
||||
assert!(
|
||||
existing_tagging.tag_set.is_empty(),
|
||||
"tagging should be empty after removing all swift-meta-* tags"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tag_preservation_no_existing_tags() {
|
||||
// Test adding Swift metadata when no tags exist
|
||||
let existing_tagging = Tagging { tag_set: vec![] };
|
||||
|
||||
let mut new_metadata = std::collections::HashMap::new();
|
||||
new_metadata.insert("color".to_string(), "blue".to_string());
|
||||
let mut new_tagging = swift_metadata_to_s3_tags(&new_metadata).unwrap();
|
||||
|
||||
let mut merged = existing_tagging.clone();
|
||||
merged.tag_set.append(&mut new_tagging.tag_set);
|
||||
|
||||
// Verify: should have only the new Swift tag
|
||||
assert_eq!(merged.tag_set.len(), 1);
|
||||
assert_eq!(merged.tag_set[0].key.as_deref(), Some("swift-meta-color"));
|
||||
assert_eq!(merged.tag_set[0].value.as_deref(), Some("blue"));
|
||||
}
|
||||
|
||||
// Object Versioning Tests
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
|
||||
use super::container;
|
||||
use super::dlo;
|
||||
use super::metadata_update::MetadataUpdate;
|
||||
use super::object;
|
||||
use super::slo;
|
||||
use super::tempurl;
|
||||
@@ -322,15 +321,32 @@ async fn handle_authenticated_request(
|
||||
Err(SwiftError::NotImplemented("Swift Account HEAD operation not yet implemented".to_string()))
|
||||
}
|
||||
Method::POST => {
|
||||
// Account metadata update. Additive, per Swift: the request
|
||||
// names the items to set (X-Account-Meta-*) and the items to
|
||||
// drop (X-Remove-Account-Meta-*, or an empty value), and
|
||||
// everything it does not name keeps its stored value. The
|
||||
// TempURL signing key lives here, so a replacing POST would
|
||||
// invalidate every outstanding signature for the account.
|
||||
let update = MetadataUpdate::from_account_headers(&headers);
|
||||
// Account metadata update - extract headers
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
|
||||
super::account::update_account_metadata(&account, &update, &credentials_opt).await?;
|
||||
// Extract X-Account-Meta-* headers
|
||||
for (key, value) in &headers {
|
||||
let key_str = key.as_str();
|
||||
if let Some(meta_key) = key_str.strip_prefix("x-account-meta-") {
|
||||
// Strip "x-account-meta-"
|
||||
if let Ok(value_str) = value.to_str() {
|
||||
metadata.insert(meta_key.to_string(), value_str.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Special handling for TempURL key headers
|
||||
// X-Account-Meta-Temp-URL-Key or X-Account-Meta-Temp-Url-Key
|
||||
if let Some(tempurl_key) = headers
|
||||
.get("x-account-meta-temp-url-key")
|
||||
.or_else(|| headers.get("x-account-meta-temp-Url-key"))
|
||||
&& let Ok(key_str) = tempurl_key.to_str()
|
||||
{
|
||||
metadata.insert("temp-url-key".to_string(), key_str.to_string());
|
||||
}
|
||||
|
||||
// Update account metadata
|
||||
super::account::update_account_metadata(&account, &metadata, &credentials_opt).await?;
|
||||
|
||||
let trans_id = generate_trans_id();
|
||||
Response::builder()
|
||||
@@ -565,15 +581,17 @@ async fn handle_authenticated_request(
|
||||
container::set_container_acl(&account, &container, new_read, new_write, &credentials).await?;
|
||||
}
|
||||
|
||||
// Update container metadata. Additive, per Swift: items the
|
||||
// request does not name keep their stored value, and removal
|
||||
// is explicit (X-Remove-Container-Meta-*, or an empty value).
|
||||
// Still called when the request named no item — an ACL-only
|
||||
// or versioning-only POST — because this is what reports 404
|
||||
// for a container that does not exist.
|
||||
let update = MetadataUpdate::from_container_headers(&headers);
|
||||
// Update container metadata - now we have access to request headers
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
for (name, value) in headers.iter() {
|
||||
if let Some(meta_key) = name.as_str().strip_prefix("x-container-meta-")
|
||||
&& let Ok(value_str) = value.to_str()
|
||||
{
|
||||
metadata.insert(meta_key.to_string(), value_str.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
container::update_container_metadata(&account, &container, &credentials, update).await?;
|
||||
container::update_container_metadata(&account, &container, &credentials, metadata).await?;
|
||||
|
||||
let trans_id = generate_trans_id();
|
||||
Response::builder()
|
||||
|
||||
@@ -1,410 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Account and container metadata updates
|
||||
//!
|
||||
//! Swift account and container POSTs are *additive*: an item the request does
|
||||
//! not mention keeps its stored value, and removal is explicit — either
|
||||
//! `X-Remove-{Account,Container}-Meta-{name}`, or the item sent with an empty
|
||||
//! value. Object POST is the one that replaces the whole set; `swift::object`
|
||||
//! handles that separately and must keep doing so.
|
||||
//!
|
||||
//! Getting this wrong is not a cosmetic divergence. Account metadata holds the
|
||||
//! TempURL signing key, so a POST that set an unrelated item while dropping
|
||||
//! the rest would invalidate every outstanding TempURL and FormPost signature
|
||||
//! for the account.
|
||||
//!
|
||||
//! This module turns a request's headers into the items to write and the items
|
||||
//! to drop, and applies that to the bucket tag set the metadata is persisted
|
||||
//! in.
|
||||
|
||||
use super::{MAX_METADATA_COUNT, MAX_METADATA_VALUE_SIZE, SwiftError, SwiftResult};
|
||||
use axum::http::HeaderMap;
|
||||
use s3s::dto::{Tag, Tagging};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
/// Request header prefix carrying an account metadata item.
|
||||
const ACCOUNT_META_HEADER_PREFIX: &str = "x-account-meta-";
|
||||
/// Request header prefix removing an account metadata item.
|
||||
const ACCOUNT_META_REMOVE_HEADER_PREFIX: &str = "x-remove-account-meta-";
|
||||
/// Request header prefix carrying a container metadata item.
|
||||
const CONTAINER_META_HEADER_PREFIX: &str = "x-container-meta-";
|
||||
/// Request header prefix removing a container metadata item.
|
||||
const CONTAINER_META_REMOVE_HEADER_PREFIX: &str = "x-remove-container-meta-";
|
||||
|
||||
/// Bucket-tag namespace holding account metadata items.
|
||||
pub(crate) const ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
|
||||
/// Bucket-tag namespace holding container metadata items.
|
||||
///
|
||||
/// Deliberately narrower than the `swift-` tags around it: the container ACL
|
||||
/// (`swift-acl-*`) and versioning (`swift-versions-location`) tags share this
|
||||
/// tag set and must survive a metadata POST.
|
||||
pub(crate) const CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
|
||||
|
||||
/// The metadata changes carried by one account or container POST.
|
||||
///
|
||||
/// Item names are held lowercased. Swift metadata names are case-insensitive,
|
||||
/// HTTP header names arrive lowercased anyway, and a removal has to match the
|
||||
/// name a previous POST stored — so normalizing once here is what makes
|
||||
/// `X-Remove-Container-Meta-Color` find a stored `color`.
|
||||
///
|
||||
/// Ordered rather than hashed so the persisted tag set — and therefore the
|
||||
/// serialized XML — comes out in a stable order for a given update.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct MetadataUpdate {
|
||||
/// Items to write, by name.
|
||||
items: BTreeMap<String, String>,
|
||||
/// Names to drop.
|
||||
removals: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl MetadataUpdate {
|
||||
/// Write `name` = `value`.
|
||||
pub fn set(mut self, name: &str, value: &str) -> Self {
|
||||
let name = name.to_lowercase();
|
||||
self.removals.remove(&name);
|
||||
self.items.insert(name, value.to_string());
|
||||
self
|
||||
}
|
||||
|
||||
/// Drop `name`, if it is stored.
|
||||
pub fn remove(mut self, name: &str) -> Self {
|
||||
let name = name.to_lowercase();
|
||||
self.items.remove(&name);
|
||||
self.removals.insert(name);
|
||||
self
|
||||
}
|
||||
|
||||
/// Parse the metadata headers of an account POST.
|
||||
pub(crate) fn from_account_headers(headers: &HeaderMap) -> Self {
|
||||
Self::from_headers(headers, ACCOUNT_META_HEADER_PREFIX, ACCOUNT_META_REMOVE_HEADER_PREFIX)
|
||||
}
|
||||
|
||||
/// Parse the metadata headers of a container POST.
|
||||
pub(crate) fn from_container_headers(headers: &HeaderMap) -> Self {
|
||||
Self::from_headers(headers, CONTAINER_META_HEADER_PREFIX, CONTAINER_META_REMOVE_HEADER_PREFIX)
|
||||
}
|
||||
|
||||
/// Parse metadata headers under `item_prefix`, and removals under
|
||||
/// `remove_prefix`. Both prefixes are lowercase, matching how `http`
|
||||
/// normalizes header names.
|
||||
///
|
||||
/// An item sent with an empty value is a removal — the deletion path
|
||||
/// Swift clients use when they do not send a dedicated remove header.
|
||||
/// A name carried by both header forms is removed: the explicit removal
|
||||
/// wins, as it does in Swift, where a remove header is rewritten into an
|
||||
/// empty-valued item header.
|
||||
fn from_headers(headers: &HeaderMap, item_prefix: &str, remove_prefix: &str) -> Self {
|
||||
let mut update = Self::default();
|
||||
|
||||
for (name, value) in headers {
|
||||
let Some(item) = name.as_str().strip_prefix(item_prefix) else {
|
||||
continue;
|
||||
};
|
||||
// A value that is not valid UTF-8 cannot be stored as a tag.
|
||||
// Skipping it matches how the rest of the Swift handlers treat
|
||||
// unreadable header values.
|
||||
let Ok(value) = value.to_str() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
update = if value.is_empty() {
|
||||
update.remove(item)
|
||||
} else {
|
||||
update.set(item, value)
|
||||
};
|
||||
}
|
||||
|
||||
for name in headers.keys() {
|
||||
if let Some(item) = name.as_str().strip_prefix(remove_prefix) {
|
||||
update = update.remove(item);
|
||||
}
|
||||
}
|
||||
|
||||
update
|
||||
}
|
||||
|
||||
/// Whether this update changes anything.
|
||||
///
|
||||
/// An ACL-only or versioning-only container POST produces an empty update:
|
||||
/// it names no metadata item, so it must leave stored metadata alone.
|
||||
pub(crate) fn is_empty(&self) -> bool {
|
||||
self.items.is_empty() && self.removals.is_empty()
|
||||
}
|
||||
|
||||
/// Reject oversized values before anything is persisted.
|
||||
///
|
||||
/// The item *count* is not checked here — an additive POST has to be
|
||||
/// measured against the merged result, which only [`Self::apply_to_tags`]
|
||||
/// can see.
|
||||
pub(crate) fn validate(&self) -> SwiftResult<()> {
|
||||
for (name, value) in &self.items {
|
||||
if value.len() > MAX_METADATA_VALUE_SIZE {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Metadata value for '{}' too large: {} bytes (max: {} bytes)",
|
||||
name,
|
||||
value.len(),
|
||||
MAX_METADATA_VALUE_SIZE
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Merge this update into the persisted tag set.
|
||||
///
|
||||
/// `prefix` is the tag namespace holding the items. Tags outside it — the
|
||||
/// container ACL and versioning tags, plus any S3 tags the bucket carries
|
||||
/// — are untouched, and so are the namespaced items this update does not
|
||||
/// name.
|
||||
///
|
||||
/// The item-count cap is checked against the merged result rather than the
|
||||
/// request: because POSTs are additive, a client could otherwise walk past
|
||||
/// it one header at a time. This runs inside the bucket metadata write
|
||||
/// guard, so the count it checks is the one about to be persisted.
|
||||
pub(crate) fn apply_to_tags(&self, current: Option<&Tagging>, prefix: &str) -> SwiftResult<Tagging> {
|
||||
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
tagging.tag_set.retain(|tag| match item_name(tag, prefix) {
|
||||
Some(name) => !self.items.contains_key(name) && !self.removals.contains(name),
|
||||
None => true,
|
||||
});
|
||||
|
||||
let merged = tagging.tag_set.iter().filter(|tag| item_name(tag, prefix).is_some()).count() + self.items.len();
|
||||
if merged > MAX_METADATA_COUNT {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Too many metadata headers: {} (max: {})",
|
||||
merged, MAX_METADATA_COUNT
|
||||
)));
|
||||
}
|
||||
|
||||
for (name, value) in &self.items {
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some(format!("{}{}", prefix, name)),
|
||||
value: Some(value.clone()),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tagging)
|
||||
}
|
||||
}
|
||||
|
||||
/// The metadata item name a tag carries, or `None` if the tag does not belong
|
||||
/// to this namespace.
|
||||
fn item_name<'a>(tag: &'a Tag, prefix: &str) -> Option<&'a str> {
|
||||
tag.key.as_deref()?.strip_prefix(prefix)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::http::{HeaderName, HeaderValue};
|
||||
|
||||
fn headers(pairs: &[(&str, &str)]) -> HeaderMap {
|
||||
let mut map = HeaderMap::new();
|
||||
for (name, value) in pairs {
|
||||
map.insert(
|
||||
HeaderName::from_bytes(name.as_bytes()).expect("test header name should parse"),
|
||||
HeaderValue::from_str(value).expect("test header value should parse"),
|
||||
);
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
fn tags(pairs: &[(&str, &str)]) -> Tagging {
|
||||
Tagging {
|
||||
tag_set: pairs
|
||||
.iter()
|
||||
.map(|(key, value)| Tag {
|
||||
key: Some((*key).to_string()),
|
||||
value: Some((*value).to_string()),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn tag_value<'a>(tagging: &'a Tagging, key: &str) -> Option<&'a str> {
|
||||
tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.find(|tag| tag.key.as_deref() == Some(key))
|
||||
.and_then(|tag| tag.value.as_deref())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_headers_collects_items_and_ignores_unrelated_headers() {
|
||||
let update = MetadataUpdate::from_container_headers(&headers(&[
|
||||
("x-container-meta-color", "blue"),
|
||||
("x-container-read", ".r:*"),
|
||||
("content-type", "text/plain"),
|
||||
]));
|
||||
|
||||
assert_eq!(update, MetadataUpdate::default().set("color", "blue"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_headers_lowercases_item_names() {
|
||||
// `http` normalizes header names on the way in, so the mixed case a
|
||||
// client sends is already gone by the time a handler sees it; the
|
||||
// lowercasing here is what makes a directly built update match.
|
||||
let update = MetadataUpdate::from_container_headers(&headers(&[("X-Container-Meta-Color", "Blue")]));
|
||||
|
||||
assert_eq!(update, MetadataUpdate::default().set("COLOR", "Blue"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_value_is_a_removal() {
|
||||
let update = MetadataUpdate::from_container_headers(&headers(&[("x-container-meta-color", "")]));
|
||||
|
||||
assert_eq!(update, MetadataUpdate::default().remove("color"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_header_drops_the_item() {
|
||||
let update = MetadataUpdate::from_account_headers(&headers(&[("x-remove-account-meta-temp-url-key", "x")]));
|
||||
|
||||
assert_eq!(update, MetadataUpdate::default().remove("temp-url-key"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_header_wins_over_a_value_for_the_same_item() {
|
||||
let update = MetadataUpdate::from_container_headers(&headers(&[
|
||||
("x-container-meta-color", "blue"),
|
||||
("x-remove-container-meta-color", "x"),
|
||||
]));
|
||||
|
||||
assert_eq!(update, MetadataUpdate::default().remove("color"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_removal_header_is_not_mistaken_for_an_item() {
|
||||
// "x-remove-container-meta-color" must not also parse as the item
|
||||
// "remove-container-meta-color" or similar.
|
||||
let update = MetadataUpdate::from_container_headers(&headers(&[("x-remove-container-meta-color", "x")]));
|
||||
|
||||
assert!(update.items.is_empty(), "a removal header must not set an item");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_with_no_metadata_headers_is_empty() {
|
||||
assert!(MetadataUpdate::from_container_headers(&headers(&[("x-container-read", ".r:*")])).is_empty());
|
||||
assert!(MetadataUpdate::default().is_empty());
|
||||
assert!(!MetadataUpdate::default().remove("color").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_preserves_items_the_update_does_not_name() {
|
||||
let current = tags(&[
|
||||
("swift-meta-color", "blue"),
|
||||
("swift-meta-season", "summer"),
|
||||
("swift-acl-read", ".r:*"),
|
||||
("swift-versions-location", "archive"),
|
||||
("unrelated-s3-tag", "keep"),
|
||||
]);
|
||||
|
||||
let merged = MetadataUpdate::default()
|
||||
.set("mood", "calm")
|
||||
.apply_to_tags(Some(¤t), CONTAINER_META_TAG_PREFIX)
|
||||
.expect("merge should be accepted");
|
||||
|
||||
assert_eq!(tag_value(&merged, "swift-meta-color"), Some("blue"));
|
||||
assert_eq!(tag_value(&merged, "swift-meta-season"), Some("summer"));
|
||||
assert_eq!(tag_value(&merged, "swift-meta-mood"), Some("calm"));
|
||||
assert_eq!(tag_value(&merged, "swift-acl-read"), Some(".r:*"));
|
||||
assert_eq!(tag_value(&merged, "swift-versions-location"), Some("archive"));
|
||||
assert_eq!(tag_value(&merged, "unrelated-s3-tag"), Some("keep"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_overwrites_a_named_item_exactly_once() {
|
||||
let current = tags(&[("swift-meta-color", "blue")]);
|
||||
|
||||
let merged = MetadataUpdate::default()
|
||||
.set("color", "red")
|
||||
.apply_to_tags(Some(¤t), CONTAINER_META_TAG_PREFIX)
|
||||
.expect("merge should be accepted");
|
||||
|
||||
assert_eq!(merged.tag_set.len(), 1, "overwriting must not duplicate the tag");
|
||||
assert_eq!(tag_value(&merged, "swift-meta-color"), Some("red"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_drops_only_the_removed_item() {
|
||||
let current = tags(&[
|
||||
("swift-meta-color", "blue"),
|
||||
("swift-meta-season", "summer"),
|
||||
("swift-acl-read", ".r:*"),
|
||||
]);
|
||||
|
||||
let merged = MetadataUpdate::default()
|
||||
.remove("color")
|
||||
.apply_to_tags(Some(¤t), CONTAINER_META_TAG_PREFIX)
|
||||
.expect("merge should be accepted");
|
||||
|
||||
assert_eq!(tag_value(&merged, "swift-meta-color"), None);
|
||||
assert_eq!(tag_value(&merged, "swift-meta-season"), Some("summer"));
|
||||
assert_eq!(tag_value(&merged, "swift-acl-read"), Some(".r:*"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_to_an_untagged_bucket_starts_from_nothing() {
|
||||
let merged = MetadataUpdate::default()
|
||||
.set("color", "blue")
|
||||
.apply_to_tags(None, ACCOUNT_META_TAG_PREFIX)
|
||||
.expect("merge should be accepted");
|
||||
|
||||
assert_eq!(tag_value(&merged, "swift-account-meta-color"), Some("blue"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_caps_the_merged_item_count_not_the_request() {
|
||||
let current = Tagging {
|
||||
tag_set: (0..MAX_METADATA_COUNT)
|
||||
.map(|i| Tag {
|
||||
key: Some(format!("{}item{}", CONTAINER_META_TAG_PREFIX, i)),
|
||||
value: Some("v".to_string()),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
|
||||
// One more item than the container already stores: rejected, even
|
||||
// though the request itself carries a single header.
|
||||
let err = MetadataUpdate::default()
|
||||
.set("overflow", "v")
|
||||
.apply_to_tags(Some(¤t), CONTAINER_META_TAG_PREFIX)
|
||||
.expect_err("exceeding the item cap must be rejected");
|
||||
assert!(matches!(err, SwiftError::BadRequest(_)), "expected BadRequest, got {err:?}");
|
||||
|
||||
// Overwriting an item already counted stays at the cap.
|
||||
MetadataUpdate::default()
|
||||
.set("item0", "v2")
|
||||
.apply_to_tags(Some(¤t), CONTAINER_META_TAG_PREFIX)
|
||||
.expect("overwriting an existing item must not trip the cap");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_oversized_values() {
|
||||
let err = MetadataUpdate::default()
|
||||
.set("color", &"b".repeat(MAX_METADATA_VALUE_SIZE + 1))
|
||||
.validate()
|
||||
.expect_err("an oversized value must be rejected");
|
||||
assert!(matches!(err, SwiftError::BadRequest(_)), "expected BadRequest, got {err:?}");
|
||||
|
||||
MetadataUpdate::default()
|
||||
.set("color", &"b".repeat(MAX_METADATA_VALUE_SIZE))
|
||||
.validate()
|
||||
.expect("a value at the limit must be accepted");
|
||||
}
|
||||
}
|
||||
@@ -44,7 +44,6 @@ pub mod expiration;
|
||||
pub mod expiration_worker;
|
||||
pub mod formpost;
|
||||
pub mod handler;
|
||||
pub mod metadata_update;
|
||||
pub mod object;
|
||||
pub mod quota;
|
||||
pub mod router;
|
||||
@@ -58,56 +57,11 @@ pub mod types;
|
||||
pub mod versioning;
|
||||
|
||||
pub use errors::{SwiftError, SwiftResult};
|
||||
pub use metadata_update::MetadataUpdate;
|
||||
pub use router::{SwiftRoute, SwiftRouter};
|
||||
|
||||
/// Maximum number of metadata headers allowed per resource (Swift standard)
|
||||
pub(crate) const MAX_METADATA_COUNT: usize = 90;
|
||||
|
||||
/// Maximum size in bytes for a single metadata value (Swift standard)
|
||||
pub(crate) const MAX_METADATA_VALUE_SIZE: usize = 256;
|
||||
|
||||
/// Validate metadata against Swift limits
|
||||
///
|
||||
/// Checks that:
|
||||
/// - Total number of metadata entries doesn't exceed MAX_METADATA_COUNT
|
||||
/// - Individual metadata values don't exceed MAX_METADATA_VALUE_SIZE
|
||||
///
|
||||
/// This is the object-metadata form, where a POST replaces the whole set and
|
||||
/// the request is therefore the complete result. Account and container POSTs
|
||||
/// are additive, so their count has to be measured against the merged state
|
||||
/// instead — see [`MetadataUpdate::apply_to_tags`].
|
||||
///
|
||||
/// Returns error if limits are exceeded.
|
||||
pub(crate) fn validate_metadata(metadata: &std::collections::HashMap<String, String>) -> SwiftResult<()> {
|
||||
// Check total metadata count
|
||||
if metadata.len() > MAX_METADATA_COUNT {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Too many metadata headers: {} (max: {})",
|
||||
metadata.len(),
|
||||
MAX_METADATA_COUNT
|
||||
)));
|
||||
}
|
||||
|
||||
// Check individual value sizes
|
||||
for (key, value) in metadata.iter() {
|
||||
if value.len() > MAX_METADATA_VALUE_SIZE {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Metadata value for '{}' too large: {} bytes (max: {} bytes)",
|
||||
key,
|
||||
value.len(),
|
||||
MAX_METADATA_VALUE_SIZE
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Note: Container, Object, and SwiftMetadata types used by Swift implementation
|
||||
pub use storage_api::public_api::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
|
||||
pub(crate) use storage_api::public_api::{
|
||||
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
|
||||
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, set_swift_bucket_metadata,
|
||||
};
|
||||
#[allow(unused_imports)]
|
||||
pub use types::{Container, Object, SwiftMetadata};
|
||||
|
||||
@@ -53,7 +53,7 @@ use super::account::validate_account_access;
|
||||
use super::container::ContainerMapper;
|
||||
use super::expiration_worker::{track_object_expiration, untrack_object_expiration};
|
||||
use super::storage_api::object::{BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
||||
use super::{SwiftError, SwiftResult, resolve_swift_object_store_handle, validate_metadata};
|
||||
use super::{SwiftError, SwiftResult, resolve_swift_object_store_handle};
|
||||
use axum::http::HeaderMap;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_rio::HashReader;
|
||||
@@ -68,6 +68,12 @@ const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object";
|
||||
const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state";
|
||||
const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at";
|
||||
|
||||
/// Maximum number of metadata headers allowed per object (Swift standard)
|
||||
const MAX_METADATA_COUNT: usize = 90;
|
||||
|
||||
/// Maximum size in bytes for a single metadata value (Swift standard)
|
||||
const MAX_METADATA_VALUE_SIZE: usize = 256;
|
||||
|
||||
/// Maximum object size in bytes (5GB - Swift default)
|
||||
const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
|
||||
|
||||
@@ -206,9 +212,10 @@ impl ObjectKeyMapper {
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn normalize_path(object: &str) -> String {
|
||||
// Split by '/', filter out empty segments (except if it's the end)
|
||||
let segments: Vec<&str> = object.split('/').collect();
|
||||
let has_trailing_slash = object.ends_with('/');
|
||||
|
||||
let normalized_segments: Vec<&str> = object.split('/').filter(|s| !s.is_empty()).collect();
|
||||
let normalized_segments: Vec<&str> = segments.into_iter().filter(|s| !s.is_empty()).collect();
|
||||
|
||||
let mut result = normalized_segments.join("/");
|
||||
|
||||
@@ -227,6 +234,38 @@ impl Default for ObjectKeyMapper {
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate metadata against Swift limits
|
||||
///
|
||||
/// Checks that:
|
||||
/// - Total number of metadata entries doesn't exceed MAX_METADATA_COUNT
|
||||
/// - Individual metadata values don't exceed MAX_METADATA_VALUE_SIZE
|
||||
///
|
||||
/// Returns error if limits are exceeded.
|
||||
fn validate_metadata(metadata: &HashMap<String, String>) -> SwiftResult<()> {
|
||||
// Check total metadata count
|
||||
if metadata.len() > MAX_METADATA_COUNT {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Too many metadata headers: {} (max: {})",
|
||||
metadata.len(),
|
||||
MAX_METADATA_COUNT
|
||||
)));
|
||||
}
|
||||
|
||||
// Check individual value sizes
|
||||
for (key, value) in metadata.iter() {
|
||||
if value.len() > MAX_METADATA_VALUE_SIZE {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Metadata value for '{}' too large: {} bytes (max: {} bytes)",
|
||||
key,
|
||||
value.len(),
|
||||
MAX_METADATA_VALUE_SIZE
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn metadata_delete_at(metadata: &HashMap<String, String>) -> Option<u64> {
|
||||
metadata
|
||||
.get(SWIFT_DELETE_AT_METADATA)
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
use super::storage_api::large_object::HTTPRangeSpec;
|
||||
use super::{SwiftError, object};
|
||||
use axum::http::{HeaderMap, Response, StatusCode};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::Body;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -82,9 +81,9 @@ impl SLOManifest {
|
||||
etag_concat.push_str(etag);
|
||||
}
|
||||
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(etag_concat.as_bytes());
|
||||
format!("\"{}-{}\"", hex::encode(hasher.finalize()), self.segments.len())
|
||||
// Calculate MD5 hash
|
||||
let hash = md5::compute(etag_concat.as_bytes());
|
||||
format!("\"{:x}-{}\"", hash, self.segments.len())
|
||||
}
|
||||
|
||||
/// Validate manifest against actual segments
|
||||
|
||||
@@ -15,19 +15,14 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_ecstore::api::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata as SwiftBucketMetadata;
|
||||
use rustfs_ecstore::api::bucket::metadata_sys::{get as get_swift_bucket_metadata_from_backend, update_config_with};
|
||||
use rustfs_ecstore::api::bucket::utils::serialize as serialize_bucket_config;
|
||||
use rustfs_ecstore::api::error::Error as SwiftStorageError;
|
||||
use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
get as get_swift_bucket_metadata_from_backend, set_bucket_metadata as set_swift_bucket_metadata_in_backend,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::error::Result as SwiftStorageResult;
|
||||
use rustfs_ecstore::api::notification::get_global_notification_sys;
|
||||
pub(crate) use rustfs_ecstore::api::runtime::object_store_handle as resolve_swift_object_store_handle;
|
||||
use rustfs_ecstore::api::storage::ECStore as SwiftStore;
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
use s3s::dto::Tagging;
|
||||
|
||||
use super::{SwiftError, SwiftResult};
|
||||
|
||||
pub(crate) mod account {
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, MakeBucketOptions};
|
||||
@@ -50,7 +45,7 @@ pub(crate) mod object {
|
||||
pub(crate) mod public_api {
|
||||
pub use super::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
|
||||
pub(crate) use super::{
|
||||
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
|
||||
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, set_swift_bucket_metadata,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -58,15 +53,6 @@ pub(crate) mod versioning {
|
||||
pub(crate) use super::storage_contracts::{ListOperations, ObjectOperations};
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
|
||||
const LOG_SUBSYSTEM_SWIFT_STORAGE: &str = "swift_storage";
|
||||
const EVENT_SWIFT_BUCKET_TAGGING_UPDATE: &str = "swift_bucket_tagging_update";
|
||||
|
||||
/// Marks the refusal to rewrite an unreadable persisted tagging config, so the
|
||||
/// caller can turn it into an actionable client error rather than a generic
|
||||
/// storage failure. Carried through the ecstore error, which is a string type.
|
||||
const UNREADABLE_TAGGING_SENTINEL: &str = "swift: persisted tagging config could not be parsed";
|
||||
|
||||
pub type SwiftGetObjectReader = <SwiftStore as storage_contracts::ObjectIO>::GetObjectReader;
|
||||
pub type SwiftObjectInfo = <SwiftStore as storage_contracts::ObjectOperations>::ObjectInfo;
|
||||
pub type SwiftObjectOptions = <SwiftStore as storage_contracts::ObjectOperations>::ObjectOptions;
|
||||
@@ -76,101 +62,8 @@ pub(crate) async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResul
|
||||
get_swift_bucket_metadata_from_backend(bucket).await
|
||||
}
|
||||
|
||||
/// Rewrite the bucket's tagging config through the persisting
|
||||
/// bucket-metadata path.
|
||||
///
|
||||
/// `rewrite` sees the tag set currently persisted on disk (`None` when the
|
||||
/// bucket has none) and returns the full replacement; an empty tag set
|
||||
/// clears the config. The read-modify-write runs under the bucket metadata
|
||||
/// system's write guard — serialized against every other config update —
|
||||
/// and the result is written to the bucket metadata file before the cache
|
||||
/// is refreshed, so a Swift metadata POST survives process restarts and
|
||||
/// disk-truth reloads. Peers are then told to reload, matching what the S3
|
||||
/// handlers do after a config write.
|
||||
///
|
||||
/// A rewrite may also refuse the update outright, for limits that can only be
|
||||
/// judged against the state being merged into. That verdict is the client's
|
||||
/// answer, so it is returned as-is rather than folded into a storage error.
|
||||
///
|
||||
/// Storage failures are logged in full and reported to the client as a
|
||||
/// generic error: these now carry real disk and quorum detail, which does not
|
||||
/// belong in a Swift response body. The one exception is an unreadable
|
||||
/// persisted config, which is reported specifically because the operator has
|
||||
/// to act on it.
|
||||
pub(crate) async fn update_swift_bucket_tagging<F>(bucket: String, rewrite: F) -> SwiftResult<()>
|
||||
where
|
||||
F: FnOnce(Option<&Tagging>) -> SwiftResult<Tagging> + Send,
|
||||
{
|
||||
// Carries a rewrite's refusal back out past the storage error the config
|
||||
// write has to fail with to abort the transaction.
|
||||
let mut rejected = None;
|
||||
|
||||
let result = update_config_with(&bucket, BUCKET_TAGGING_CONFIG, |bm| {
|
||||
// Merging onto an unparseable tag set would silently drop every tag
|
||||
// the bucket has — including the container ACL and versioning tags —
|
||||
// because the rewrite closures treat "no parsed tags" as "no tags".
|
||||
// Refuse instead: the persisted config is intact, just unreadable.
|
||||
if !bm.tagging_config_xml.is_empty() && bm.tagging_config.is_none() {
|
||||
return Err(SwiftStorageError::other(UNREADABLE_TAGGING_SENTINEL));
|
||||
}
|
||||
|
||||
let tagging = match rewrite(bm.tagging_config.as_ref()) {
|
||||
Ok(tagging) => tagging,
|
||||
Err(err) => {
|
||||
rejected = Some(err);
|
||||
return Err(SwiftStorageError::other("swift: tagging rewrite rejected the update"));
|
||||
}
|
||||
};
|
||||
|
||||
if tagging.tag_set.is_empty() {
|
||||
Ok(Vec::new())
|
||||
} else {
|
||||
// The S3 XML serializer, not quick_xml: the metadata loader's
|
||||
// parse step must be able to round-trip what we persist.
|
||||
serialize_bucket_config(&tagging)
|
||||
.map_err(|e| SwiftStorageError::other(format!("failed to serialize bucket tagging: {e}")))
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
// Nothing was written, but that is the rewrite's own decision about the
|
||||
// request rather than a storage fault, so it is not logged as one.
|
||||
if let Some(err) = rejected {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Err(err) = result {
|
||||
let unreadable = err.to_string().contains(UNREADABLE_TAGGING_SENTINEL);
|
||||
tracing::error!(
|
||||
event = EVENT_SWIFT_BUCKET_TAGGING_UPDATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SWIFT_STORAGE,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason = if unreadable { "unreadable_persisted_config" } else { "storage_failure" },
|
||||
result = "failed",
|
||||
"swift bucket tagging update failed"
|
||||
);
|
||||
// A Swift-only client has no way to repair this itself, so say what
|
||||
// happened and name the remedy instead of a bare storage error.
|
||||
return Err(if unreadable {
|
||||
SwiftError::Conflict(format!(
|
||||
"The persisted tagging configuration for container store '{bucket}' cannot be parsed, so metadata cannot be updated without discarding it. Reset it with the S3 DeleteBucketTagging API."
|
||||
))
|
||||
} else {
|
||||
SwiftError::InternalServerError("Metadata update operation failed".to_string())
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(notification_sys) = get_global_notification_sys() {
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = notification_sys.load_bucket_metadata(&bucket).await {
|
||||
tracing::warn!(bucket = %bucket, error = %err, "failed to notify peers after swift bucket tagging update");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
pub(crate) async fn set_swift_bucket_metadata(bucket: String, metadata: SwiftBucketMetadata) -> SwiftStorageResult<()> {
|
||||
set_swift_bucket_metadata_in_backend(bucket, metadata).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_swift_bucket_usage() -> SwiftStorageResult<Option<HashMap<String, (u64, u64)>>> {
|
||||
|
||||
@@ -222,7 +222,7 @@ where
|
||||
created: modified,
|
||||
is_dir: false,
|
||||
etag: output.e_tag.as_ref().map(etag_to_string),
|
||||
content_type: output.content_type,
|
||||
content_type: output.content_type.map(|c| c.to_string()),
|
||||
}) as Box<dyn DavMetaData>)
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -1185,7 +1185,7 @@ where
|
||||
created: modified,
|
||||
is_dir: false,
|
||||
etag: output.e_tag.as_ref().map(etag_to_string),
|
||||
content_type: output.content_type,
|
||||
content_type: output.content_type.map(|c| c.to_string()),
|
||||
}) as Box<dyn DavMetaData>)
|
||||
}
|
||||
ResolvedPath::Directory { metadata, .. } => {
|
||||
@@ -1204,7 +1204,7 @@ where
|
||||
created: modified,
|
||||
is_dir: true,
|
||||
etag: metadata.as_ref().and_then(|output| output.e_tag.as_ref().map(etag_to_string)),
|
||||
content_type: metadata.and_then(|output| output.content_type),
|
||||
content_type: metadata.and_then(|output| output.content_type.map(|c| c.to_string())),
|
||||
}) as Box<dyn DavMetaData>)
|
||||
}
|
||||
};
|
||||
@@ -2035,7 +2035,7 @@ mod tests {
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<ListObjectsV2Output, Self::Error> {
|
||||
let prefix = input.prefix.unwrap_or_default();
|
||||
let prefix = input.prefix.map(|p| p.to_string()).unwrap_or_default();
|
||||
let mut keys: Vec<String> = self
|
||||
.state
|
||||
.lock()
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! ECStore facade boundary for the protocols integration tests.
|
||||
//!
|
||||
//! The Swift tests need to drive bucket-metadata reloads the way the peer
|
||||
//! `LoadBucketMetadata` RPC does. Everything they touch from `rustfs_ecstore`
|
||||
//! is aliased here so the tests themselves hold no raw facade subpaths, the
|
||||
//! same boundary `crates/protocols/src/swift/storage_api.rs` provides for the
|
||||
//! Swift implementation.
|
||||
|
||||
pub use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
|
||||
pub use rustfs_ecstore::api::bucket::metadata_sys::{get as get_bucket_metadata, set_bucket_metadata};
|
||||
pub use rustfs_ecstore::api::runtime::object_store_handle;
|
||||
@@ -436,10 +436,12 @@ fn test_symlink_to_nested_object() {
|
||||
fn test_listing_empty_container() {
|
||||
let objects: Vec<&str> = vec![];
|
||||
|
||||
assert!(objects.is_empty());
|
||||
let filtered: Vec<_> = objects.iter().collect();
|
||||
assert_eq!(filtered.len(), 0);
|
||||
|
||||
// With prefix
|
||||
assert!(!objects.iter().any(|o| o.starts_with("prefix/")));
|
||||
let with_prefix: Vec<_> = objects.iter().filter(|o| o.starts_with("prefix/")).collect();
|
||||
assert_eq!(with_prefix.len(), 0);
|
||||
}
|
||||
|
||||
/// Test listing lexicographic ordering
|
||||
|
||||
@@ -1,461 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression tests: a Swift metadata POST must be persisted to the bucket
|
||||
//! metadata file, not just the in-memory cache. The metadata has to survive
|
||||
//! the disk-truth reloads performed by peer LoadBucketMetadata notifications
|
||||
//! and the periodic refresh loop — and, transitively, a process restart.
|
||||
//!
|
||||
//! Durability is what makes the *content* of those writes matter, so this file
|
||||
//! also covers Swift's additive account/container POST semantics: an item the
|
||||
//! request does not name keeps its stored value, and removal is explicit. The
|
||||
//! two are tested together because a reload is the only way to tell a real
|
||||
//! merge from one that happened to look right in the cache.
|
||||
|
||||
#![cfg(feature = "swift")]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_protocols::swift::container::{ContainerMapper, update_container_metadata};
|
||||
use rustfs_protocols::swift::{MetadataUpdate, SwiftError, account, container};
|
||||
use rustfs_test_utils::TestECStoreEnv;
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
mod ecstore_test_compat;
|
||||
use ecstore_test_compat::{get_bucket_metadata, load_bucket_metadata, object_store_handle, set_bucket_metadata};
|
||||
|
||||
fn keystone_credentials(project_id: &str) -> Credentials {
|
||||
let mut claims = HashMap::new();
|
||||
claims.insert("keystone_project_id".to_string(), json!(project_id));
|
||||
claims.insert("keystone_roles".to_string(), json!(["member"]));
|
||||
|
||||
Credentials {
|
||||
access_key: "keystone:swift-test".to_string(),
|
||||
claims: Some(claims),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// The account-metadata bucket name scheme from `swift::account`
|
||||
/// (`swift-account-{sha256(account)[0..16]}`), mirrored here so the test can
|
||||
/// reload that bucket's metadata from disk.
|
||||
fn account_metadata_bucket_name(account: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(account.as_bytes());
|
||||
let hash = hex::encode(hasher.finalize());
|
||||
format!("swift-account-{}", &hash[0..16])
|
||||
}
|
||||
|
||||
/// Replace the cached bucket metadata with what is actually on disk — the
|
||||
/// same thing a peer LoadBucketMetadata notification or the periodic refresh
|
||||
/// loop does. Before the fix this silently discarded every Swift metadata
|
||||
/// POST, because those writes only ever touched the cache.
|
||||
async fn reload_bucket_metadata_from_disk(bucket: &str) {
|
||||
let store = object_store_handle().expect("test store should be published");
|
||||
let bm = load_bucket_metadata(store, bucket)
|
||||
.await
|
||||
.expect("bucket metadata should load from disk");
|
||||
set_bucket_metadata(bucket.to_string(), bm)
|
||||
.await
|
||||
.expect("reloaded metadata should install");
|
||||
}
|
||||
|
||||
/// The `swift-meta-*` tags currently persisted for a container, keyed the way
|
||||
/// a Swift client sees them. `get_container_metadata` would be the natural
|
||||
/// reader, but it additionally requires a data-usage snapshot for the object
|
||||
/// count and byte total, which a bare test store has none of — and that is
|
||||
/// orthogonal to whether the metadata itself was persisted.
|
||||
async fn persisted_container_metadata(bucket: &str) -> HashMap<String, String> {
|
||||
let bm = get_bucket_metadata(bucket).await.expect("bucket metadata should be cached");
|
||||
let mut out = HashMap::new();
|
||||
if let Some(tagging) = &bm.tagging_config {
|
||||
for tag in &tagging.tag_set {
|
||||
if let (Some(key), Some(value)) = (&tag.key, &tag.value)
|
||||
&& let Some(meta_key) = key.strip_prefix("swift-meta-")
|
||||
{
|
||||
out.insert(meta_key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Every scenario that needs a store runs against ONE environment: the Swift
|
||||
/// handlers resolve the ambient object store and bucket-metadata system, so a
|
||||
/// second `TestECStoreEnv` in this process would race the first.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn swift_metadata_writes_are_durable() {
|
||||
let env = TestECStoreEnv::builder().prefix("swift_meta_persist").build().await;
|
||||
|
||||
posts_survive_disk_truth_reload(&env).await;
|
||||
tag_writers_preserve_each_others_state(&env).await;
|
||||
unrelated_account_posts_preserve_the_tempurl_key().await;
|
||||
acl_only_posts_leave_container_metadata_alone(&env).await;
|
||||
versioning_writes_reject_missing_containers().await;
|
||||
}
|
||||
|
||||
async fn posts_survive_disk_truth_reload(env: &TestECStoreEnv) {
|
||||
// --- Container metadata POST (X-Container-Meta-*) ---
|
||||
let project_id = "swiftpersistproj";
|
||||
let swift_account = format!("AUTH_{project_id}");
|
||||
let credentials = keystone_credentials(project_id);
|
||||
let swift_container = "photos";
|
||||
let bucket = ContainerMapper::default().swift_to_s3_bucket(swift_container, project_id);
|
||||
env.make_bucket(&bucket, false).await;
|
||||
|
||||
update_container_metadata(
|
||||
&swift_account,
|
||||
swift_container,
|
||||
&credentials,
|
||||
MetadataUpdate::default().set("color", "blue"),
|
||||
)
|
||||
.await
|
||||
.expect("container metadata POST should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
let container_meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(
|
||||
container_meta.get("color").map(String::as_str),
|
||||
Some("blue"),
|
||||
"container metadata POST must survive a disk-truth metadata reload"
|
||||
);
|
||||
|
||||
// A follow-up POST names a different item. Swift POSTs are additive, so
|
||||
// the new item lands and the first one stays — and both are still there
|
||||
// after a reload, which is what proves the merge ran against disk state
|
||||
// rather than looking right in the cache.
|
||||
update_container_metadata(
|
||||
&swift_account,
|
||||
swift_container,
|
||||
&credentials,
|
||||
MetadataUpdate::default().set("season", "summer"),
|
||||
)
|
||||
.await
|
||||
.expect("second container metadata POST should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
let container_meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(container_meta.get("season").map(String::as_str), Some("summer"));
|
||||
assert_eq!(
|
||||
container_meta.get("color").map(String::as_str),
|
||||
Some("blue"),
|
||||
"a POST that does not name an item must not delete it"
|
||||
);
|
||||
|
||||
// X-Remove-Container-Meta-Color is the deletion path, and it has to be
|
||||
// durable in the other direction: the removed item must not come back
|
||||
// from the persisted tags on reload.
|
||||
update_container_metadata(&swift_account, swift_container, &credentials, MetadataUpdate::default().remove("color"))
|
||||
.await
|
||||
.expect("container metadata removal should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
let container_meta = persisted_container_metadata(&bucket).await;
|
||||
assert!(
|
||||
!container_meta.contains_key("color"),
|
||||
"an explicitly removed item must not resurrect on reload"
|
||||
);
|
||||
assert_eq!(
|
||||
container_meta.get("season").map(String::as_str),
|
||||
Some("summer"),
|
||||
"removing one item must not disturb the others"
|
||||
);
|
||||
|
||||
// --- Container versioning POST (X-Versions-Location) ---
|
||||
let archive_container = "photos-archive";
|
||||
let archive_bucket = ContainerMapper::default().swift_to_s3_bucket(archive_container, project_id);
|
||||
env.make_bucket(&archive_bucket, false).await;
|
||||
|
||||
container::enable_versioning(&swift_account, swift_container, archive_container, &credentials)
|
||||
.await
|
||||
.expect("enable versioning should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
let location = container::get_versions_location(&swift_account, swift_container, &credentials)
|
||||
.await
|
||||
.expect("versions location should load");
|
||||
assert_eq!(
|
||||
location.as_deref(),
|
||||
Some(archive_container),
|
||||
"versions location must survive a disk-truth metadata reload"
|
||||
);
|
||||
|
||||
// --- Account metadata POST (TempURL keys etc.) ---
|
||||
account::update_account_metadata(
|
||||
&swift_account,
|
||||
&MetadataUpdate::default().set("temp-url-key", "s3cr3t"),
|
||||
&Some(credentials.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("account metadata POST should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&account_metadata_bucket_name(&swift_account)).await;
|
||||
|
||||
let loaded = account::get_account_metadata(&swift_account, &None)
|
||||
.await
|
||||
.expect("account metadata should load");
|
||||
assert_eq!(
|
||||
loaded.get("temp-url-key").map(String::as_str),
|
||||
Some("s3cr3t"),
|
||||
"account metadata POST must survive a disk-truth metadata reload"
|
||||
);
|
||||
}
|
||||
|
||||
/// The rewrites all share one tag set, so a closure that ignored the current
|
||||
/// state would still pass a single-feature test. Drive ACLs, versioning and
|
||||
/// container metadata over the same container and assert each survives the
|
||||
/// others — and that clearing one leaves the rest alone.
|
||||
async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
|
||||
let project_id = "swiftcrosstagproj";
|
||||
let swift_account = format!("AUTH_{project_id}");
|
||||
let credentials = keystone_credentials(project_id);
|
||||
let container = "shared";
|
||||
let archive = "shared-archive";
|
||||
let bucket = ContainerMapper::default().swift_to_s3_bucket(container, project_id);
|
||||
env.make_bucket(&bucket, false).await;
|
||||
env.make_bucket(&ContainerMapper::default().swift_to_s3_bucket(archive, project_id), false)
|
||||
.await;
|
||||
|
||||
update_container_metadata(&swift_account, container, &credentials, MetadataUpdate::default().set("color", "blue"))
|
||||
.await
|
||||
.expect("container metadata POST should succeed");
|
||||
container::enable_versioning(&swift_account, container, archive, &credentials)
|
||||
.await
|
||||
.expect("enable versioning should succeed");
|
||||
container::set_container_acl(&swift_account, container, Some(".r:*"), Some("AUTH_other"), &credentials)
|
||||
.await
|
||||
.expect("set container ACL should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
// All three writers' state coexists after a disk-truth reload.
|
||||
let meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(meta.get("color").map(String::as_str), Some("blue"));
|
||||
assert_eq!(
|
||||
container::get_versions_location(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("versions location should load")
|
||||
.as_deref(),
|
||||
Some(archive)
|
||||
);
|
||||
let acl = container::get_container_acl(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("container ACL should load");
|
||||
assert!(!acl.read.is_empty(), "read ACL must survive the reload");
|
||||
assert!(!acl.write.is_empty(), "write ACL must survive the reload");
|
||||
|
||||
// Disabling versioning drops only the versioning tag.
|
||||
container::disable_versioning(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("disable versioning should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
assert_eq!(
|
||||
container::get_versions_location(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("versions location should load"),
|
||||
None,
|
||||
"disable_versioning must clear the versioning tag durably"
|
||||
);
|
||||
let meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(
|
||||
meta.get("color").map(String::as_str),
|
||||
Some("blue"),
|
||||
"disable_versioning must not disturb container metadata"
|
||||
);
|
||||
let acl = container::get_container_acl(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("container ACL should load");
|
||||
assert!(!acl.read.is_empty(), "disable_versioning must not disturb the ACL");
|
||||
}
|
||||
|
||||
/// The failure additive POSTs exist to prevent. Account metadata holds the
|
||||
/// TempURL signing key, so a POST that replaced the whole set — setting a
|
||||
/// quota, say — would delete that key and permanently invalidate every
|
||||
/// outstanding TempURL and FormPost signature for the account. Durable
|
||||
/// storage made that loss permanent instead of a cache blip, so the check
|
||||
/// runs against reloaded disk state.
|
||||
///
|
||||
/// Relies on the ambient store the caller's `TestECStoreEnv` published; the
|
||||
/// account metadata bucket is created by the write path itself.
|
||||
async fn unrelated_account_posts_preserve_the_tempurl_key() {
|
||||
let project_id = "swiftmergeproj";
|
||||
let swift_account = format!("AUTH_{project_id}");
|
||||
let credentials = Some(keystone_credentials(project_id));
|
||||
let account_bucket = account_metadata_bucket_name(&swift_account);
|
||||
|
||||
account::update_account_metadata(&swift_account, &MetadataUpdate::default().set("temp-url-key", "s3cr3t"), &credentials)
|
||||
.await
|
||||
.expect("TempURL key POST should succeed");
|
||||
|
||||
// A later POST that has nothing to do with the TempURL key.
|
||||
account::update_account_metadata(&swift_account, &MetadataUpdate::default().set("quota-bytes", "100"), &credentials)
|
||||
.await
|
||||
.expect("quota POST should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&account_bucket).await;
|
||||
|
||||
let loaded = account::get_account_metadata(&swift_account, &None)
|
||||
.await
|
||||
.expect("account metadata should load");
|
||||
assert_eq!(
|
||||
loaded.get("temp-url-key").map(String::as_str),
|
||||
Some("s3cr3t"),
|
||||
"an unrelated account POST must not delete the TempURL signing key"
|
||||
);
|
||||
assert_eq!(loaded.get("quota-bytes").map(String::as_str), Some("100"));
|
||||
|
||||
// And the lookup that actually breaks when the key is lost still resolves.
|
||||
assert_eq!(
|
||||
account::get_tempurl_key(&swift_account, &None)
|
||||
.await
|
||||
.expect("TempURL key should load")
|
||||
.as_deref(),
|
||||
Some("s3cr3t"),
|
||||
"TempURL signature validation must still find the key"
|
||||
);
|
||||
|
||||
// X-Remove-Account-Meta-Quota-Bytes drops that item and nothing else.
|
||||
account::update_account_metadata(&swift_account, &MetadataUpdate::default().remove("quota-bytes"), &credentials)
|
||||
.await
|
||||
.expect("account metadata removal should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&account_bucket).await;
|
||||
|
||||
let loaded = account::get_account_metadata(&swift_account, &None)
|
||||
.await
|
||||
.expect("account metadata should load");
|
||||
assert!(
|
||||
!loaded.contains_key("quota-bytes"),
|
||||
"an explicitly removed item must not resurrect on reload"
|
||||
);
|
||||
assert_eq!(
|
||||
loaded.get("temp-url-key").map(String::as_str),
|
||||
Some("s3cr3t"),
|
||||
"removing one item must not disturb the TempURL key"
|
||||
);
|
||||
}
|
||||
|
||||
/// An ACL-only or versioning-only container POST carries no `X-Container-Meta-*`
|
||||
/// header, but the handler still runs the metadata step on every POST. That
|
||||
/// step must leave stored metadata alone rather than treating "named nothing"
|
||||
/// as "wants everything gone".
|
||||
async fn acl_only_posts_leave_container_metadata_alone(env: &TestECStoreEnv) {
|
||||
let project_id = "swiftaclonlyproj";
|
||||
let swift_account = format!("AUTH_{project_id}");
|
||||
let credentials = keystone_credentials(project_id);
|
||||
let container = "gallery";
|
||||
let bucket = ContainerMapper::default().swift_to_s3_bucket(container, project_id);
|
||||
env.make_bucket(&bucket, false).await;
|
||||
|
||||
update_container_metadata(&swift_account, container, &credentials, MetadataUpdate::default().set("color", "blue"))
|
||||
.await
|
||||
.expect("container metadata POST should succeed");
|
||||
|
||||
// What the handler does for `POST … -H 'X-Container-Read: .r:*'`: set the
|
||||
// ACL, then run the metadata step with an update that names no item.
|
||||
container::set_container_acl(&swift_account, container, Some(".r:*"), None, &credentials)
|
||||
.await
|
||||
.expect("ACL-only POST should succeed");
|
||||
update_container_metadata(&swift_account, container, &credentials, MetadataUpdate::default())
|
||||
.await
|
||||
.expect("the metadata step of an ACL-only POST should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
assert_eq!(
|
||||
persisted_container_metadata(&bucket).await.get("color").map(String::as_str),
|
||||
Some("blue"),
|
||||
"an ACL-only POST must not wipe X-Container-Meta-*"
|
||||
);
|
||||
let acl = container::get_container_acl(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("container ACL should load");
|
||||
assert!(!acl.read.is_empty(), "the ACL that POST set must be stored");
|
||||
}
|
||||
|
||||
/// A container that does not exist must not get metadata persisted for it:
|
||||
/// the metadata loader turns "nothing on disk" into a fresh default, so an
|
||||
/// unguarded rewrite would create an orphan metadata file and cache a
|
||||
/// fabricated default as authoritative.
|
||||
async fn versioning_writes_reject_missing_containers() {
|
||||
let project_id = "swiftmissingproj";
|
||||
let swift_account = format!("AUTH_{project_id}");
|
||||
let credentials = keystone_credentials(project_id);
|
||||
let missing = "no-such-container";
|
||||
|
||||
let err = container::disable_versioning(&swift_account, missing, &credentials)
|
||||
.await
|
||||
.expect_err("disabling versioning on a missing container must fail");
|
||||
assert!(
|
||||
matches!(err, SwiftError::NotFound(_)),
|
||||
"expected NotFound for a missing container, got {err:?}"
|
||||
);
|
||||
|
||||
// Naming no item skips the persisted write, but must not skip the
|
||||
// existence check that turns a POST to a missing container into a 404.
|
||||
let err = update_container_metadata(&swift_account, missing, &credentials, MetadataUpdate::default())
|
||||
.await
|
||||
.expect_err("a metadata POST to a missing container must fail");
|
||||
assert!(
|
||||
matches!(err, SwiftError::NotFound(_)),
|
||||
"expected NotFound for a missing container, got {err:?}"
|
||||
);
|
||||
|
||||
let bucket = ContainerMapper::default().swift_to_s3_bucket(missing, project_id);
|
||||
assert!(
|
||||
get_bucket_metadata(&bucket).await.is_err(),
|
||||
"a rejected write must not have cached metadata for a nonexistent container"
|
||||
);
|
||||
}
|
||||
|
||||
/// Account metadata holds the account's TempURL signing key, and it is now
|
||||
/// durable — so a write for someone else's account would be a persistent,
|
||||
/// cluster-wide takeover of that account's pre-signed URLs, not a cache blip.
|
||||
/// The write path must reject both a foreign account and a missing token,
|
||||
/// while reads stay open for pre-auth TempURL signature validation.
|
||||
///
|
||||
/// Deliberately builds no store: both rejections must happen before the write
|
||||
/// path resolves storage at all, and a second `TestECStoreEnv` in this process
|
||||
/// would race the other test over the ambient store handle.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn account_metadata_write_rejects_foreign_and_anonymous_callers() {
|
||||
let victim_account = "AUTH_victimproject";
|
||||
let attacker_credentials = keystone_credentials("attackerproject");
|
||||
|
||||
let poisoned = MetadataUpdate::default().set("temp-url-key", "attacker-key");
|
||||
|
||||
let err = account::update_account_metadata(victim_account, &poisoned, &Some(attacker_credentials))
|
||||
.await
|
||||
.expect_err("writing another account's metadata must be rejected");
|
||||
assert!(
|
||||
matches!(err, SwiftError::Forbidden(_)),
|
||||
"cross-account metadata write must be Forbidden, got {err:?}"
|
||||
);
|
||||
|
||||
let err = account::update_account_metadata(victim_account, &poisoned, &None)
|
||||
.await
|
||||
.expect_err("anonymous account metadata write must be rejected");
|
||||
assert!(
|
||||
matches!(err, SwiftError::Unauthorized(_)),
|
||||
"anonymous metadata write must be Unauthorized, got {err:?}"
|
||||
);
|
||||
}
|
||||
@@ -484,59 +484,6 @@ pub struct DeletePathsResponse {
|
||||
pub error: ::core::option::Option<Error>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct SnapshotLeaseRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub path: ::prost::alloc::string::String,
|
||||
#[prost(uint64, tag = "4")]
|
||||
pub ttl_ms: u64,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct SnapshotLeaseRenewRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub path: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub token: ::prost::bytes::Bytes,
|
||||
#[prost(uint64, tag = "5")]
|
||||
pub ttl_ms: u64,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct SnapshotLeaseReleaseRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub path: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub token: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct SnapshotLeaseResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(bytes = "bytes", tag = "2")]
|
||||
pub token: ::prost::bytes::Bytes,
|
||||
#[prost(uint32, tag = "3")]
|
||||
pub protocol_version: u32,
|
||||
#[prost(message, optional, tag = "4")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct SnapshotLeaseMutationResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ReadMetadataRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
@@ -1648,51 +1595,6 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "Delete"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn acquire_snapshot_lease(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SnapshotLeaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/AcquireSnapshotLease");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "AcquireSnapshotLease"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn renew_snapshot_lease(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SnapshotLeaseRenewRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenewSnapshotLease");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenewSnapshotLease"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn release_snapshot_lease(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SnapshotLeaseReleaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseMutationResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReleaseSnapshotLease");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ReleaseSnapshotLease"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn verify_file(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::VerifyFileRequest>,
|
||||
@@ -2882,18 +2784,6 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::DeleteRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::DeleteResponse>, tonic::Status>;
|
||||
async fn acquire_snapshot_lease(
|
||||
&self,
|
||||
request: tonic::Request<super::SnapshotLeaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status>;
|
||||
async fn renew_snapshot_lease(
|
||||
&self,
|
||||
request: tonic::Request<super::SnapshotLeaseRenewRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status>;
|
||||
async fn release_snapshot_lease(
|
||||
&self,
|
||||
request: tonic::Request<super::SnapshotLeaseReleaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseMutationResponse>, tonic::Status>;
|
||||
async fn verify_file(
|
||||
&self,
|
||||
request: tonic::Request<super::VerifyFileRequest>,
|
||||
@@ -3536,90 +3426,6 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/AcquireSnapshotLease" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct AcquireSnapshotLeaseSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::SnapshotLeaseRequest> for AcquireSnapshotLeaseSvc<T> {
|
||||
type Response = super::SnapshotLeaseResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::SnapshotLeaseRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::acquire_snapshot_lease(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = AcquireSnapshotLeaseSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenewSnapshotLease" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenewSnapshotLeaseSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::SnapshotLeaseRenewRequest> for RenewSnapshotLeaseSvc<T> {
|
||||
type Response = super::SnapshotLeaseResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::SnapshotLeaseRenewRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::renew_snapshot_lease(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = RenewSnapshotLeaseSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/ReleaseSnapshotLease" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ReleaseSnapshotLeaseSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::SnapshotLeaseReleaseRequest> for ReleaseSnapshotLeaseSvc<T> {
|
||||
type Response = super::SnapshotLeaseMutationResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::SnapshotLeaseReleaseRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::release_snapshot_lease(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ReleaseSnapshotLeaseSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/VerifyFile" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct VerifyFileSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
+1
-454
@@ -447,23 +447,10 @@ impl CanonicalBodyBuilder {
|
||||
self.push_bytes(field.as_bytes())
|
||||
}
|
||||
|
||||
fn push_optional_str(&mut self, field: Option<&str>) -> Result<(), std::num::TryFromIntError> {
|
||||
self.body.push(u8::from(field.is_some()));
|
||||
self.push_str(field.unwrap_or_default())
|
||||
}
|
||||
|
||||
fn push_bool(&mut self, field: bool) {
|
||||
self.body.push(u8::from(field));
|
||||
}
|
||||
|
||||
fn push_u32(&mut self, field: u32) {
|
||||
self.body.extend_from_slice(&field.to_be_bytes());
|
||||
}
|
||||
|
||||
fn push_u64(&mut self, field: u64) {
|
||||
self.body.extend_from_slice(&field.to_be_bytes());
|
||||
}
|
||||
|
||||
fn push_count(&mut self, count: usize) -> Result<(), std::num::TryFromIntError> {
|
||||
self.body.extend_from_slice(&u64::try_from(count)?.to_be_bytes());
|
||||
Ok(())
|
||||
@@ -474,211 +461,6 @@ impl CanonicalBodyBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
pub const PEER_RESTSIGNAL: &str = "signal";
|
||||
pub const PEER_RESTSUB_SYS: &str = "sub-sys";
|
||||
pub const PEER_RESTDRY_RUN: &str = "dry-run";
|
||||
|
||||
/// A stable semantic body for a side-effecting unary RPC.
|
||||
///
|
||||
/// Implementations deliberately enumerate handler-consumed fields instead of re-encoding the
|
||||
/// protobuf message, whose unknown fields and map order are not a mixed-version contract.
|
||||
pub trait CanonicalMutationBody {
|
||||
fn canonical_body(&self) -> Result<Vec<u8>, std::num::TryFromIntError>;
|
||||
}
|
||||
|
||||
macro_rules! impl_canonical_mutation_body {
|
||||
($request:ty, $domain:expr, |$value:ident, $body:ident| $fields:block) => {
|
||||
impl CanonicalMutationBody for $request {
|
||||
fn canonical_body(&self) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let $value = self;
|
||||
let mut $body = CanonicalBodyBuilder::new($domain);
|
||||
$fields
|
||||
Ok($body.finish())
|
||||
}
|
||||
}
|
||||
};
|
||||
($request:ty, $domain:expr) => {
|
||||
impl CanonicalMutationBody for $request {
|
||||
fn canonical_body(&self) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
Ok(CanonicalBodyBuilder::new($domain).finish())
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::SignalServiceRequest,
|
||||
b"rustfs-signal-service-request-v1\0",
|
||||
|request, body| {
|
||||
let vars = request.vars.as_ref().map(|vars| &vars.value);
|
||||
body.push_optional_str(vars.and_then(|vars| vars.get(PEER_RESTSIGNAL).map(String::as_str)))?;
|
||||
body.push_optional_str(vars.and_then(|vars| vars.get(PEER_RESTSUB_SYS).map(String::as_str)))?;
|
||||
body.push_optional_str(vars.and_then(|vars| vars.get(PEER_RESTDRY_RUN).map(String::as_str)))?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::HealBucketRequest,
|
||||
b"rustfs-heal-bucket-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.options)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::MakeBucketRequest,
|
||||
b"rustfs-make-bucket-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.name)?;
|
||||
body.push_str(&request.options)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::DeleteBucketRequest,
|
||||
b"rustfs-delete-bucket-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.options)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::GenerallyLockRequest,
|
||||
b"rustfs-lock-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.args)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::BatchGenerallyLockRequest,
|
||||
b"rustfs-lock-batch-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_count(request.args.len())?;
|
||||
for arg in &request.args {
|
||||
body.push_str(arg)?;
|
||||
}
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::LoadBucketMetadataRequest,
|
||||
b"rustfs-load-bucket-metadata-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_bool(request.scanner_maintenance_change);
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::DeleteBucketMetadataRequest,
|
||||
b"rustfs-delete-bucket-metadata-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.bucket)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::DeletePolicyRequest,
|
||||
b"rustfs-delete-policy-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.policy_name)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::LoadPolicyRequest,
|
||||
b"rustfs-load-policy-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.policy_name)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::LoadPolicyMappingRequest,
|
||||
b"rustfs-load-policy-mapping-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.user_or_group)?;
|
||||
body.push_u64(request.user_type);
|
||||
body.push_bool(request.is_group);
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::DeleteUserRequest,
|
||||
b"rustfs-delete-user-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.access_key)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::DeleteServiceAccountRequest,
|
||||
b"rustfs-delete-service-account-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.access_key)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::LoadUserRequest,
|
||||
b"rustfs-load-user-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.access_key)?;
|
||||
body.push_bool(request.temp);
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::LoadServiceAccountRequest,
|
||||
b"rustfs-load-service-account-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.access_key)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::LoadGroupRequest,
|
||||
b"rustfs-load-group-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.group)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::ReloadSiteReplicationConfigRequest,
|
||||
b"rustfs-reload-site-replication-config-request-v1\0"
|
||||
);
|
||||
impl_canonical_mutation_body!(proto_gen::node_service::ReloadPoolMetaRequest, b"rustfs-reload-pool-meta-request-v1\0");
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::StopRebalanceRequest,
|
||||
b"rustfs-stop-rebalance-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_str(&request.expected_rebalance_id)?;
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::LoadRebalanceMetaRequest,
|
||||
b"rustfs-load-rebalance-meta-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_bool(request.start_rebalance);
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::StartDecommissionRequest,
|
||||
b"rustfs-start-decommission-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_count(request.pool_indices.len())?;
|
||||
for pool_index in &request.pool_indices {
|
||||
body.push_u32(*pool_index);
|
||||
}
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::CancelDecommissionRequest,
|
||||
b"rustfs-cancel-decommission-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_u32(request.pool_index);
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::ClearDecommissionRequest,
|
||||
b"rustfs-clear-decommission-request-v1\0",
|
||||
|request, body| {
|
||||
body.push_u32(request.pool_index);
|
||||
}
|
||||
);
|
||||
impl_canonical_mutation_body!(
|
||||
proto_gen::node_service::LoadTransitionTierConfigRequest,
|
||||
b"rustfs-load-transition-tier-config-request-v1\0"
|
||||
);
|
||||
|
||||
// Canonical request bodies for the mutating NodeService disk RPCs (backlog#1327 body-digest
|
||||
// binding). Each covers every semantic wire field — including both the msgpack `_bin` payload and
|
||||
// its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
|
||||
@@ -793,40 +575,6 @@ pub fn canonical_delete_paths_request_body(
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_snapshot_lease_request_body(
|
||||
request: &proto_gen::node_service::SnapshotLeaseRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-snapshot-lease-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_u64(request.ttl_ms);
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_snapshot_lease_renew_request_body(
|
||||
request: &proto_gen::node_service::SnapshotLeaseRenewRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-snapshot-lease-renew-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
body.push_u64(request.ttl_ms);
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_snapshot_lease_release_request_body(
|
||||
request: &proto_gen::node_service::SnapshotLeaseReleaseRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-snapshot-lease-release-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_file_request_body(
|
||||
request: &proto_gen::node_service::RenameFileRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
@@ -914,8 +662,7 @@ mod disk_mutation_canonical_tests {
|
||||
use super::proto_gen::node_service::{
|
||||
DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
SettlePartTransactionRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
@@ -1273,58 +1020,6 @@ mod disk_mutation_canonical_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_lease_canonical_bodies_bind_every_field() {
|
||||
let acquire = SnapshotLeaseRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
ttl_ms: 60_000,
|
||||
};
|
||||
let mut acquire_bodies = vec![canonical_snapshot_lease_request_body(&acquire).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut SnapshotLeaseRequest| r.disk = "d2".into(),
|
||||
|r: &mut SnapshotLeaseRequest| r.volume = "v2".into(),
|
||||
|r: &mut SnapshotLeaseRequest| r.path = "p2".into(),
|
||||
|r: &mut SnapshotLeaseRequest| r.ttl_ms = 60_001,
|
||||
] {
|
||||
let mut request = acquire.clone();
|
||||
mutate(&mut request);
|
||||
acquire_bodies.push(canonical_snapshot_lease_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&acquire_bodies);
|
||||
|
||||
let renew = SnapshotLeaseRenewRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
token: vec![1; 16].into(),
|
||||
ttl_ms: 60_000,
|
||||
};
|
||||
let mut changed_token = renew.clone();
|
||||
changed_token.token = vec![2; 16].into();
|
||||
let mut changed_ttl = renew.clone();
|
||||
changed_ttl.ttl_ms += 1;
|
||||
assert_all_distinct(&[
|
||||
canonical_snapshot_lease_renew_request_body(&renew).unwrap(),
|
||||
canonical_snapshot_lease_renew_request_body(&changed_token).unwrap(),
|
||||
canonical_snapshot_lease_renew_request_body(&changed_ttl).unwrap(),
|
||||
]);
|
||||
|
||||
let release = SnapshotLeaseReleaseRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut changed_release = release.clone();
|
||||
changed_release.token = vec![2; 16].into();
|
||||
assert_ne!(
|
||||
canonical_snapshot_lease_release_request_body(&release).unwrap(),
|
||||
canonical_snapshot_lease_release_request_body(&changed_release).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_mutation_canonical_domains_are_distinct_per_message() {
|
||||
// The same field values must never authenticate one RPC's request as another's.
|
||||
@@ -1350,154 +1045,6 @@ mod disk_mutation_canonical_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod non_disk_mutation_canonical_tests {
|
||||
use super::*;
|
||||
use proto_gen::node_service::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
macro_rules! assert_fields_bound {
|
||||
($request:ty, {$($field:ident: $value:expr),+ $(,)?}) => {{
|
||||
let baseline = <$request>::default();
|
||||
let expected = baseline.canonical_body().expect("baseline canonical body should encode");
|
||||
$(
|
||||
let mut variant = baseline.clone();
|
||||
variant.$field = $value;
|
||||
assert_ne!(
|
||||
expected,
|
||||
variant.canonical_body().expect("variant canonical body should encode"),
|
||||
concat!(stringify!($request), " omitted field ", stringify!($field)),
|
||||
);
|
||||
)+
|
||||
}};
|
||||
}
|
||||
|
||||
fn signal_request(signal: Option<&str>, sub_system: Option<&str>, dry_run: Option<&str>) -> SignalServiceRequest {
|
||||
let mut value = HashMap::new();
|
||||
if let Some(signal) = signal {
|
||||
value.insert(PEER_RESTSIGNAL.to_string(), signal.to_string());
|
||||
}
|
||||
if let Some(sub_system) = sub_system {
|
||||
value.insert(PEER_RESTSUB_SYS.to_string(), sub_system.to_string());
|
||||
}
|
||||
if let Some(dry_run) = dry_run {
|
||||
value.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string());
|
||||
}
|
||||
SignalServiceRequest {
|
||||
vars: Some(Mss { value }),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_service_canonical_body_is_versioned_and_binds_every_semantic_field() {
|
||||
let request = signal_request(Some("2"), Some("scanner"), Some("false"));
|
||||
let baseline = request.canonical_body().expect("small signal request should encode");
|
||||
assert!(baseline.starts_with(b"rustfs-signal-service-request-v1\0"));
|
||||
|
||||
for variant in [
|
||||
signal_request(Some("1"), Some("scanner"), Some("false")),
|
||||
signal_request(Some("2"), Some("heal"), Some("false")),
|
||||
signal_request(Some("2"), Some("scanner"), Some("true")),
|
||||
signal_request(None, Some("scanner"), Some("false")),
|
||||
signal_request(Some("2"), None, Some("false")),
|
||||
signal_request(Some("2"), Some("scanner"), None),
|
||||
] {
|
||||
assert_ne!(baseline, variant.canonical_body().expect("small signal request should encode"));
|
||||
}
|
||||
|
||||
let mut ignored_field = request;
|
||||
ignored_field
|
||||
.vars
|
||||
.as_mut()
|
||||
.expect("signal vars should exist")
|
||||
.value
|
||||
.insert("future-field".to_string(), "ignored".to_string());
|
||||
assert_eq!(
|
||||
baseline,
|
||||
ignored_field
|
||||
.canonical_body()
|
||||
.expect("unknown signal field should not affect the semantic body"),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_service_canonical_body_distinguishes_missing_and_empty_fields() {
|
||||
assert_ne!(
|
||||
signal_request(None, Some("scanner"), Some("false"))
|
||||
.canonical_body()
|
||||
.expect("missing signal should encode"),
|
||||
signal_request(Some(""), Some("scanner"), Some("false"))
|
||||
.canonical_body()
|
||||
.expect("empty signal should encode"),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_and_lock_canonical_bodies_bind_every_semantic_field() {
|
||||
assert_fields_bound!(HealBucketRequest, { bucket: "bucket".into(), options: "opts".into() });
|
||||
assert_fields_bound!(MakeBucketRequest, { name: "bucket".into(), options: "opts".into() });
|
||||
assert_fields_bound!(DeleteBucketRequest, { bucket: "bucket".into(), options: "opts".into() });
|
||||
assert_fields_bound!(GenerallyLockRequest, { args: "lock".into() });
|
||||
assert_fields_bound!(BatchGenerallyLockRequest, { args: vec!["first".into(), "second".into()] });
|
||||
|
||||
let first = BatchGenerallyLockRequest {
|
||||
args: vec!["first".into(), "second".into()],
|
||||
};
|
||||
let reversed = BatchGenerallyLockRequest {
|
||||
args: vec!["second".into(), "first".into()],
|
||||
};
|
||||
assert_ne!(first.canonical_body().unwrap(), reversed.canonical_body().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_and_iam_canonical_bodies_bind_every_semantic_field() {
|
||||
assert_fields_bound!(LoadBucketMetadataRequest, {
|
||||
bucket: "bucket".into(),
|
||||
scanner_maintenance_change: true,
|
||||
});
|
||||
assert_fields_bound!(DeleteBucketMetadataRequest, { bucket: "bucket".into() });
|
||||
assert_fields_bound!(DeletePolicyRequest, { policy_name: "policy".into() });
|
||||
assert_fields_bound!(LoadPolicyRequest, { policy_name: "policy".into() });
|
||||
assert_fields_bound!(LoadPolicyMappingRequest, {
|
||||
user_or_group: "user".into(),
|
||||
user_type: 1,
|
||||
is_group: true,
|
||||
});
|
||||
assert_fields_bound!(DeleteUserRequest, { access_key: "user".into() });
|
||||
assert_fields_bound!(DeleteServiceAccountRequest, { access_key: "service".into() });
|
||||
assert_fields_bound!(LoadUserRequest, { access_key: "user".into(), temp: true });
|
||||
assert_fields_bound!(LoadServiceAccountRequest, { access_key: "service".into() });
|
||||
assert_fields_bound!(LoadGroupRequest, { group: "group".into() });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_plane_canonical_bodies_bind_every_semantic_field() {
|
||||
assert_fields_bound!(StopRebalanceRequest, { expected_rebalance_id: "rebalance".into() });
|
||||
assert_fields_bound!(LoadRebalanceMetaRequest, { start_rebalance: true });
|
||||
assert_fields_bound!(StartDecommissionRequest, { pool_indices: vec![1, 2] });
|
||||
assert_fields_bound!(CancelDecommissionRequest, { pool_index: 1 });
|
||||
assert_fields_bound!(ClearDecommissionRequest, { pool_index: 1 });
|
||||
|
||||
let first = StartDecommissionRequest {
|
||||
pool_indices: vec![1, 2],
|
||||
};
|
||||
let reversed = StartDecommissionRequest {
|
||||
pool_indices: vec![2, 1],
|
||||
};
|
||||
assert_ne!(first.canonical_body().unwrap(), reversed.canonical_body().unwrap());
|
||||
|
||||
let empty_domains = [
|
||||
ReloadSiteReplicationConfigRequest::default().canonical_body().unwrap(),
|
||||
ReloadPoolMetaRequest::default().canonical_body().unwrap(),
|
||||
LoadTransitionTierConfigRequest::default().canonical_body().unwrap(),
|
||||
];
|
||||
assert!(empty_domains.iter().all(|body| !body.is_empty()));
|
||||
assert_ne!(empty_domains[0], empty_domains[1]);
|
||||
assert_ne!(empty_domains[0], empty_domains[2]);
|
||||
assert_ne!(empty_domains[1], empty_domains[2]);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_activity_tests {
|
||||
use super::{
|
||||
|
||||
+85
-122
@@ -342,40 +342,6 @@ message DeletePathsResponse {
|
||||
optional Error error = 2;
|
||||
}
|
||||
|
||||
message SnapshotLeaseRequest {
|
||||
string disk = 1;
|
||||
string volume = 2;
|
||||
string path = 3;
|
||||
uint64 ttl_ms = 4;
|
||||
}
|
||||
|
||||
message SnapshotLeaseRenewRequest {
|
||||
string disk = 1;
|
||||
string volume = 2;
|
||||
string path = 3;
|
||||
bytes token = 4;
|
||||
uint64 ttl_ms = 5;
|
||||
}
|
||||
|
||||
message SnapshotLeaseReleaseRequest {
|
||||
string disk = 1;
|
||||
string volume = 2;
|
||||
string path = 3;
|
||||
bytes token = 4;
|
||||
}
|
||||
|
||||
message SnapshotLeaseResponse {
|
||||
bool success = 1;
|
||||
bytes token = 2;
|
||||
uint32 protocol_version = 3;
|
||||
optional Error error = 4;
|
||||
}
|
||||
|
||||
message SnapshotLeaseMutationResponse {
|
||||
bool success = 1;
|
||||
optional Error error = 2;
|
||||
}
|
||||
|
||||
message ReadMetadataRequest {
|
||||
string disk = 1;
|
||||
string volume = 2;
|
||||
@@ -988,107 +954,104 @@ message GetLiveEventsResponse {
|
||||
|
||||
service NodeService {
|
||||
/* -------------------------------meta service-------------------------- */
|
||||
rpc Ping(PingRequest) returns (PingResponse) {}; // auth-policy: read-only
|
||||
rpc HealBucket(HealBucketRequest) returns (HealBucketResponse) {}; // auth-policy: body-bound
|
||||
rpc ListBucket(ListBucketRequest) returns (ListBucketResponse) {}; // auth-policy: read-only
|
||||
rpc MakeBucket(MakeBucketRequest) returns (MakeBucketResponse) {}; // auth-policy: body-bound
|
||||
rpc GetBucketInfo(GetBucketInfoRequest) returns (GetBucketInfoResponse) {}; // auth-policy: read-only
|
||||
rpc DeleteBucket(DeleteBucketRequest) returns (DeleteBucketResponse) {}; // auth-policy: body-bound
|
||||
rpc Ping(PingRequest) returns (PingResponse) {};
|
||||
rpc HealBucket(HealBucketRequest) returns (HealBucketResponse) {};
|
||||
rpc ListBucket(ListBucketRequest) returns (ListBucketResponse) {};
|
||||
rpc MakeBucket(MakeBucketRequest) returns (MakeBucketResponse) {};
|
||||
rpc GetBucketInfo(GetBucketInfoRequest) returns (GetBucketInfoResponse) {};
|
||||
rpc DeleteBucket(DeleteBucketRequest) returns (DeleteBucketResponse) {};
|
||||
|
||||
/* -------------------------------disk service-------------------------- */
|
||||
|
||||
rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {}; // auth-policy: read-only
|
||||
rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {}; // auth-policy: body-bound
|
||||
rpc Delete(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound
|
||||
rpc AcquireSnapshotLease(SnapshotLeaseRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
|
||||
rpc RenewSnapshotLease(SnapshotLeaseRenewRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
|
||||
rpc ReleaseSnapshotLease(SnapshotLeaseReleaseRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
|
||||
rpc VerifyFile(VerifyFileRequest) returns (VerifyFileResponse) {}; // auth-policy: read-only
|
||||
rpc ReadParts(ReadPartsRequest) returns (ReadPartsResponse) {}; // auth-policy: read-only
|
||||
rpc CheckParts(CheckPartsRequest) returns (CheckPartsResponse) {}; // auth-policy: read-only
|
||||
rpc PreparePartTransaction(PreparePartTransactionRequest) returns (PreparePartTransactionResponse) {}; // auth-policy: body-bound
|
||||
rpc RenamePart(RenamePartRequest) returns (RenamePartResponse) {}; // auth-policy: body-bound
|
||||
rpc SettlePartTransaction(SettlePartTransactionRequest) returns (SettlePartTransactionResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameFile(RenameFileRequest) returns (RenameFileResponse) {}; // auth-policy: body-bound
|
||||
rpc Write(WriteRequest) returns (WriteResponse) {}; // auth-policy: unimplemented
|
||||
rpc WriteStream(stream WriteRequest) returns (stream WriteResponse) {}; // auth-policy: streaming
|
||||
rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {};
|
||||
rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {};
|
||||
rpc Delete(DeleteRequest) returns (DeleteResponse) {};
|
||||
rpc VerifyFile(VerifyFileRequest) returns (VerifyFileResponse) {};
|
||||
rpc ReadParts(ReadPartsRequest) returns (ReadPartsResponse) {};
|
||||
rpc CheckParts(CheckPartsRequest) returns (CheckPartsResponse) {};
|
||||
rpc PreparePartTransaction(PreparePartTransactionRequest) returns (PreparePartTransactionResponse) {};
|
||||
rpc RenamePart(RenamePartRequest) returns (RenamePartResponse) {};
|
||||
rpc SettlePartTransaction(SettlePartTransactionRequest) returns (SettlePartTransactionResponse) {};
|
||||
rpc RenameFile(RenameFileRequest) returns (RenameFileResponse) {};
|
||||
rpc Write(WriteRequest) returns (WriteResponse) {};
|
||||
rpc WriteStream(stream WriteRequest) returns (stream WriteResponse) {};
|
||||
// rpc Append(AppendRequest) returns (AppendResponse) {};
|
||||
rpc ReadAt(stream ReadAtRequest) returns (stream ReadAtResponse) {}; // auth-policy: streaming
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc MakeVolumes(MakeVolumesRequest) returns (MakeVolumesResponse) {}; // auth-policy: body-bound
|
||||
rpc MakeVolume(MakeVolumeRequest) returns (MakeVolumeResponse) {}; // auth-policy: body-bound
|
||||
rpc ListVolumes(ListVolumesRequest) returns (ListVolumesResponse) {}; // auth-policy: read-only
|
||||
rpc StatVolume(StatVolumeRequest) returns (StatVolumeResponse) {}; // auth-policy: read-only
|
||||
rpc DeletePaths(DeletePathsRequest) returns (DeletePathsResponse) {}; // auth-policy: body-bound
|
||||
rpc UpdateMetadata(UpdateMetadataRequest) returns (UpdateMetadataResponse) {}; // auth-policy: body-bound
|
||||
rpc ReadMetadata(ReadMetadataRequest) returns (ReadMetadataResponse) {}; // auth-policy: read-only
|
||||
rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {}; // auth-policy: body-bound
|
||||
rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {}; // auth-policy: read-only
|
||||
rpc BatchReadVersion(BatchReadVersionRequest) returns (BatchReadVersionResponse) {}; // auth-policy: read-only
|
||||
rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {}; // auth-policy: read-only
|
||||
rpc DeleteVersion(DeleteVersionRequest) returns (DeleteVersionResponse) {}; // auth-policy: body-bound
|
||||
rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {}; // auth-policy: body-bound
|
||||
rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {}; // auth-policy: read-only
|
||||
rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {}; // auth-policy: body-bound
|
||||
rpc DiskInfo(DiskInfoRequest) returns (DiskInfoResponse) {}; // auth-policy: read-only
|
||||
rpc ReadAt(stream ReadAtRequest) returns (stream ReadAtResponse) {};
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse) {};
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {};
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {};
|
||||
rpc MakeVolumes(MakeVolumesRequest) returns (MakeVolumesResponse) {};
|
||||
rpc MakeVolume(MakeVolumeRequest) returns (MakeVolumeResponse) {};
|
||||
rpc ListVolumes(ListVolumesRequest) returns (ListVolumesResponse) {};
|
||||
rpc StatVolume(StatVolumeRequest) returns (StatVolumeResponse) {};
|
||||
rpc DeletePaths(DeletePathsRequest) returns (DeletePathsResponse) {};
|
||||
rpc UpdateMetadata(UpdateMetadataRequest) returns (UpdateMetadataResponse) {};
|
||||
rpc ReadMetadata(ReadMetadataRequest) returns (ReadMetadataResponse) {};
|
||||
rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {};
|
||||
rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {};
|
||||
rpc BatchReadVersion(BatchReadVersionRequest) returns (BatchReadVersionResponse) {};
|
||||
rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {};
|
||||
rpc DeleteVersion(DeleteVersionRequest) returns (DeleteVersionResponse) {};
|
||||
rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {};
|
||||
rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {};
|
||||
rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {};
|
||||
rpc DiskInfo(DiskInfoRequest) returns (DiskInfoResponse) {};
|
||||
|
||||
|
||||
/* -------------------------------lock service-------------------------- */
|
||||
|
||||
rpc Lock(GenerallyLockRequest) returns (GenerallyLockResponse) {}; // auth-policy: body-bound
|
||||
rpc UnLock(GenerallyLockRequest) returns (GenerallyLockResponse) {}; // auth-policy: body-bound
|
||||
rpc ForceUnLock(GenerallyLockRequest) returns (GenerallyLockResponse) {}; // auth-policy: body-bound
|
||||
rpc Refresh(GenerallyLockRequest) returns (GenerallyLockResponse) {}; // auth-policy: body-bound
|
||||
rpc LockBatch(BatchGenerallyLockRequest) returns (BatchGenerallyLockResponse) {}; // auth-policy: body-bound
|
||||
rpc UnLockBatch(BatchGenerallyLockRequest) returns (BatchGenerallyLockResponse) {}; // auth-policy: body-bound
|
||||
rpc Lock(GenerallyLockRequest) returns (GenerallyLockResponse) {};
|
||||
rpc UnLock(GenerallyLockRequest) returns (GenerallyLockResponse) {};
|
||||
rpc ForceUnLock(GenerallyLockRequest) returns (GenerallyLockResponse) {};
|
||||
rpc Refresh(GenerallyLockRequest) returns (GenerallyLockResponse) {};
|
||||
rpc LockBatch(BatchGenerallyLockRequest) returns (BatchGenerallyLockResponse) {};
|
||||
rpc UnLockBatch(BatchGenerallyLockRequest) returns (BatchGenerallyLockResponse) {};
|
||||
|
||||
/* -------------------------------peer rest service-------------------------- */
|
||||
|
||||
rpc LocalStorageInfo(LocalStorageInfoRequest) returns (LocalStorageInfoResponse) {}; // auth-policy: read-only
|
||||
rpc ServerInfo(ServerInfoRequest) returns (ServerInfoResponse) {}; // auth-policy: read-only
|
||||
rpc GetCpus(GetCpusRequest) returns (GetCpusResponse) {}; // auth-policy: read-only
|
||||
rpc GetNetInfo(GetNetInfoRequest) returns (GetNetInfoResponse) {}; // auth-policy: read-only
|
||||
rpc GetPartitions(GetPartitionsRequest) returns (GetPartitionsResponse) {}; // auth-policy: read-only
|
||||
rpc GetOsInfo(GetOsInfoRequest) returns (GetOsInfoResponse) {}; // auth-policy: read-only
|
||||
rpc GetSELinuxInfo(GetSELinuxInfoRequest) returns (GetSELinuxInfoResponse) {}; // auth-policy: read-only
|
||||
rpc GetSysConfig(GetSysConfigRequest) returns (GetSysConfigResponse) {}; // auth-policy: read-only
|
||||
rpc GetSysErrors(GetSysErrorsRequest) returns (GetSysErrorsResponse) {}; // auth-policy: read-only
|
||||
rpc GetMemInfo(GetMemInfoRequest) returns (GetMemInfoResponse) {}; // auth-policy: read-only
|
||||
rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {}; // auth-policy: read-only
|
||||
rpc GetProcInfo(GetProcInfoRequest) returns (GetProcInfoResponse) {}; // auth-policy: read-only
|
||||
rpc StartProfiling(StartProfilingRequest) returns (StartProfilingResponse) {}; // auth-policy: unimplemented
|
||||
rpc DownloadProfileData(DownloadProfileDataRequest) returns (DownloadProfileDataResponse) {}; // auth-policy: unimplemented
|
||||
rpc GetBucketStats(GetBucketStatsDataRequest) returns (GetBucketStatsDataResponse) {}; // auth-policy: read-only
|
||||
rpc GetSRMetrics(GetSRMetricsDataRequest) returns (GetSRMetricsDataResponse) {}; // auth-policy: unimplemented
|
||||
rpc GetAllBucketStats(GetAllBucketStatsRequest) returns (GetAllBucketStatsResponse) {}; // auth-policy: unimplemented
|
||||
rpc LoadBucketMetadata(LoadBucketMetadataRequest) returns (LoadBucketMetadataResponse) {}; // auth-policy: body-bound
|
||||
rpc DeleteBucketMetadata(DeleteBucketMetadataRequest) returns (DeleteBucketMetadataResponse) {}; // auth-policy: body-bound
|
||||
rpc DeletePolicy(DeletePolicyRequest) returns (DeletePolicyResponse) {}; // auth-policy: body-bound
|
||||
rpc LoadPolicy(LoadPolicyRequest) returns (LoadPolicyResponse) {}; // auth-policy: body-bound
|
||||
rpc LoadPolicyMapping(LoadPolicyMappingRequest) returns (LoadPolicyMappingResponse) {}; // auth-policy: body-bound
|
||||
rpc DeleteUser(DeleteUserRequest) returns (DeleteUserResponse) {}; // auth-policy: body-bound
|
||||
rpc DeleteServiceAccount(DeleteServiceAccountRequest) returns (DeleteServiceAccountResponse) {}; // auth-policy: body-bound
|
||||
rpc LoadUser(LoadUserRequest) returns (LoadUserResponse) {}; // auth-policy: body-bound
|
||||
rpc LoadServiceAccount(LoadServiceAccountRequest) returns (LoadServiceAccountResponse) {}; // auth-policy: body-bound
|
||||
rpc LoadGroup(LoadGroupRequest) returns (LoadGroupResponse) {}; // auth-policy: body-bound
|
||||
rpc ReloadSiteReplicationConfig(ReloadSiteReplicationConfigRequest) returns (ReloadSiteReplicationConfigResponse) {}; // auth-policy: body-bound
|
||||
rpc LocalStorageInfo(LocalStorageInfoRequest) returns (LocalStorageInfoResponse) {};
|
||||
rpc ServerInfo(ServerInfoRequest) returns (ServerInfoResponse) {};
|
||||
rpc GetCpus(GetCpusRequest) returns (GetCpusResponse) {};
|
||||
rpc GetNetInfo(GetNetInfoRequest) returns (GetNetInfoResponse) {};
|
||||
rpc GetPartitions(GetPartitionsRequest) returns (GetPartitionsResponse) {};
|
||||
rpc GetOsInfo(GetOsInfoRequest) returns (GetOsInfoResponse) {};
|
||||
rpc GetSELinuxInfo(GetSELinuxInfoRequest) returns (GetSELinuxInfoResponse) {};
|
||||
rpc GetSysConfig(GetSysConfigRequest) returns (GetSysConfigResponse) {};
|
||||
rpc GetSysErrors(GetSysErrorsRequest) returns (GetSysErrorsResponse) {};
|
||||
rpc GetMemInfo(GetMemInfoRequest) returns (GetMemInfoResponse) {};
|
||||
rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {};
|
||||
rpc GetProcInfo(GetProcInfoRequest) returns (GetProcInfoResponse) {};
|
||||
rpc StartProfiling(StartProfilingRequest) returns (StartProfilingResponse) {};
|
||||
rpc DownloadProfileData(DownloadProfileDataRequest) returns (DownloadProfileDataResponse) {};
|
||||
rpc GetBucketStats(GetBucketStatsDataRequest) returns (GetBucketStatsDataResponse) {};
|
||||
rpc GetSRMetrics(GetSRMetricsDataRequest) returns (GetSRMetricsDataResponse) {};
|
||||
rpc GetAllBucketStats(GetAllBucketStatsRequest) returns (GetAllBucketStatsResponse) {};
|
||||
rpc LoadBucketMetadata(LoadBucketMetadataRequest) returns (LoadBucketMetadataResponse) {};
|
||||
rpc DeleteBucketMetadata(DeleteBucketMetadataRequest) returns (DeleteBucketMetadataResponse) {};
|
||||
rpc DeletePolicy(DeletePolicyRequest) returns (DeletePolicyResponse) {};
|
||||
rpc LoadPolicy(LoadPolicyRequest) returns (LoadPolicyResponse) {};
|
||||
rpc LoadPolicyMapping(LoadPolicyMappingRequest) returns (LoadPolicyMappingResponse) {};
|
||||
rpc DeleteUser(DeleteUserRequest) returns (DeleteUserResponse) {};
|
||||
rpc DeleteServiceAccount(DeleteServiceAccountRequest) returns (DeleteServiceAccountResponse) {};
|
||||
rpc LoadUser(LoadUserRequest) returns (LoadUserResponse) {};
|
||||
rpc LoadServiceAccount(LoadServiceAccountRequest) returns (LoadServiceAccountResponse) {};
|
||||
rpc LoadGroup(LoadGroupRequest) returns (LoadGroupResponse) {};
|
||||
rpc ReloadSiteReplicationConfig(ReloadSiteReplicationConfigRequest) returns (ReloadSiteReplicationConfigResponse) {};
|
||||
// rpc VerifyBinary() returns () {};
|
||||
// rpc CommitBinary() returns () {};
|
||||
rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {}; // auth-policy: body-bound
|
||||
rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {}; // auth-policy: body-bound
|
||||
rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {}; // auth-policy: read-only
|
||||
rpc GetMetacacheListing(GetMetacacheListingRequest) returns (GetMetacacheListingResponse) {}; // auth-policy: unimplemented
|
||||
rpc UpdateMetacacheListing(UpdateMetacacheListingRequest) returns (UpdateMetacacheListingResponse) {}; // auth-policy: unimplemented
|
||||
rpc ReloadPoolMeta(ReloadPoolMetaRequest) returns (ReloadPoolMetaResponse) {}; // auth-policy: body-bound
|
||||
rpc StopRebalance(StopRebalanceRequest) returns (StopRebalanceResponse) {}; // auth-policy: body-bound
|
||||
rpc LoadRebalanceMeta(LoadRebalanceMetaRequest) returns (LoadRebalanceMetaResponse) {}; // auth-policy: body-bound
|
||||
rpc StartDecommission(StartDecommissionRequest) returns (StartDecommissionResponse) {}; // auth-policy: body-bound
|
||||
rpc CancelDecommission(CancelDecommissionRequest) returns (CancelDecommissionResponse) {}; // auth-policy: body-bound
|
||||
rpc ClearDecommission(ClearDecommissionRequest) returns (ClearDecommissionResponse) {}; // auth-policy: body-bound
|
||||
rpc LoadTransitionTierConfig(LoadTransitionTierConfigRequest) returns (LoadTransitionTierConfigResponse) {}; // auth-policy: body-bound
|
||||
rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only
|
||||
rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {};
|
||||
rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {};
|
||||
rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {};
|
||||
rpc GetMetacacheListing(GetMetacacheListingRequest) returns (GetMetacacheListingResponse) {};
|
||||
rpc UpdateMetacacheListing(UpdateMetacacheListingRequest) returns (UpdateMetacacheListingResponse) {};
|
||||
rpc ReloadPoolMeta(ReloadPoolMetaRequest) returns (ReloadPoolMetaResponse) {};
|
||||
rpc StopRebalance(StopRebalanceRequest) returns (StopRebalanceResponse) {};
|
||||
rpc LoadRebalanceMeta(LoadRebalanceMetaRequest) returns (LoadRebalanceMetaResponse) {};
|
||||
rpc StartDecommission(StartDecommissionRequest) returns (StartDecommissionResponse) {};
|
||||
rpc CancelDecommission(CancelDecommissionRequest) returns (CancelDecommissionResponse) {};
|
||||
rpc ClearDecommission(ClearDecommissionRequest) returns (ClearDecommissionResponse) {};
|
||||
rpc LoadTransitionTierConfig(LoadTransitionTierConfigRequest) returns (LoadTransitionTierConfigResponse) {};
|
||||
rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {};
|
||||
}
|
||||
|
||||
service HealControlService {
|
||||
|
||||
@@ -47,7 +47,7 @@ rmp-serde = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-storage-api = { workspace = true }
|
||||
http = { workspace = true }
|
||||
|
||||
@@ -698,7 +698,7 @@ impl DataUsageCache {
|
||||
let mut visited = HashSet::new();
|
||||
visited.insert(hash_path(path).key());
|
||||
let mut flat = self.flatten_with_guard(root, &mut visited, 0);
|
||||
if flat.replication_stats.as_ref().is_some_and(|stats| stats.is_empty()) {
|
||||
if flat.replication_stats.as_ref().is_some_and(|stats| stats.empty()) {
|
||||
flat.replication_stats = None;
|
||||
}
|
||||
Some(flat)
|
||||
@@ -1574,7 +1574,6 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO};
|
||||
use crate::{ScannerGetObjectReader, ScannerPutObjReader};
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
|
||||
use serde_json::Value;
|
||||
use std::io::Cursor;
|
||||
use std::pin::Pin;
|
||||
@@ -2768,55 +2767,6 @@ mod tests {
|
||||
assert!(flat.children.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_recursive_prunes_empty_and_preserves_threshold_replication_stats() {
|
||||
let root = hash_path("bucket");
|
||||
let child = hash_path("bucket/child");
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace_hashed(&root, &None, &DataUsageEntry::default());
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
replication_stats: Some(ReplicationAllStats::default()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.size_recursive("bucket")
|
||||
.expect("scanner bucket usage should flatten")
|
||||
.replication_stats
|
||||
.is_none()
|
||||
);
|
||||
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:threshold".to_string(),
|
||||
ReplicationStats {
|
||||
after_threshold_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let flattened = cache.size_recursive("bucket").expect("scanner bucket usage should flatten");
|
||||
let replication = flattened
|
||||
.replication_stats
|
||||
.expect("threshold-only replication stats must survive pruning");
|
||||
|
||||
assert_eq!(replication.targets["arn:test:threshold"].after_threshold_count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_flatten_rejects_dangling_child() {
|
||||
let root_key = hash_path("bucket").key();
|
||||
|
||||
+45
-340
@@ -14,8 +14,6 @@
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use std::sync::{Arc, LazyLock, RwLock};
|
||||
|
||||
use crate::ScannerObjectIO;
|
||||
@@ -40,8 +38,8 @@ use bytes::Bytes;
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_common::heal_channel::HealScanMode;
|
||||
use rustfs_common::metrics::{
|
||||
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
|
||||
emit_scan_cycle_complete, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
|
||||
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete,
|
||||
emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
|
||||
};
|
||||
use rustfs_config::ScannerSpeed;
|
||||
#[cfg(test)]
|
||||
@@ -52,12 +50,9 @@ use rustfs_config::{
|
||||
use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
#[cfg(test)]
|
||||
use tokio::sync::Notify;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
use crate::storage_api::scan::{
|
||||
@@ -98,44 +93,6 @@ const SCANNER_CYCLE_STATE_MAGIC: &[u8; 8] = b"RSCYC001";
|
||||
const SCANNER_CYCLE_STATE_HEADER_LEN: usize = 24;
|
||||
#[cfg(test)]
|
||||
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
|
||||
#[cfg(test)]
|
||||
type ScannerCycleStatePersistTestHook = (u64, Arc<Notify>);
|
||||
#[cfg(test)]
|
||||
static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCycleStatePersistTestHook>>> =
|
||||
LazyLock::new(|| StdMutex::new(None));
|
||||
|
||||
#[cfg(test)]
|
||||
struct ScannerCycleStatePersistTestHookGuard;
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScannerCycleStatePersistTestHookGuard {
|
||||
fn drop(&mut self) {
|
||||
*SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_scanner_cycle_state_persist_test_hook(leader_epoch: u64, reached: Arc<Notify>) -> ScannerCycleStatePersistTestHookGuard {
|
||||
*SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((leader_epoch, reached));
|
||||
ScannerCycleStatePersistTestHookGuard
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn notify_scanner_cycle_state_persist_test_hook(leader_epoch: u64) {
|
||||
let reached = SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.as_ref()
|
||||
.filter(|(expected_epoch, _)| *expected_epoch == leader_epoch)
|
||||
.map(|(_, reached)| reached.clone());
|
||||
if let Some(reached) = reached {
|
||||
reached.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
enum ScannerCycleStateError {
|
||||
@@ -1734,10 +1691,10 @@ fn data_usage_persist_timeout() -> Duration {
|
||||
DataUsageCache::persistence_timeout()
|
||||
}
|
||||
|
||||
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
|
||||
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle) {
|
||||
cycle_info.current = 0;
|
||||
global_metrics().clear_current_scan_mode();
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
global_metrics().set_cycle(Some(cycle_info.clone())).await;
|
||||
}
|
||||
|
||||
fn encode_scanner_cycle_state(cycle_info: &CurrentCycle, leader_epoch: u64) -> Result<Vec<u8>, ScannerCycleStateError> {
|
||||
@@ -2261,8 +2218,6 @@ async fn persist_scanner_cycle_state(
|
||||
return false;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
notify_scanner_cycle_state_persist_test_hook(leader_epoch);
|
||||
match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
|
||||
.await
|
||||
{
|
||||
@@ -2385,6 +2340,7 @@ async fn persist_scanner_cycle_state(
|
||||
|
||||
if persisted_cycle.next >= cycle_info.next {
|
||||
*cycle_info = persisted_cycle;
|
||||
global_metrics().set_cycle(Some(cycle_info.clone())).await;
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
@@ -2450,7 +2406,6 @@ async fn finalize_partial_scan_cycle(
|
||||
cycle_info: &mut CurrentCycle,
|
||||
revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
|
||||
) -> bool {
|
||||
// A budget-limited cycle is deliberate pacing, not a failure. The cycle counter
|
||||
// must still advance (and persist) because per-bucket next_cycle is stamped from
|
||||
@@ -2467,14 +2422,11 @@ async fn finalize_partial_scan_cycle(
|
||||
error = %err,
|
||||
"Scanner partial cycle could not advance"
|
||||
);
|
||||
mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await;
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
return false;
|
||||
}
|
||||
cycle_info.current = 0;
|
||||
global_metrics().clear_current_scan_mode();
|
||||
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
persisted
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await
|
||||
}
|
||||
|
||||
async fn persist_required_scanner_cycle_floor(
|
||||
@@ -2484,7 +2436,6 @@ async fn persist_required_scanner_cycle_floor(
|
||||
revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
required_cycle: u64,
|
||||
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
|
||||
) -> bool {
|
||||
if required_cycle <= cycle_info.current || required_cycle == u64::MAX {
|
||||
error!(
|
||||
@@ -2497,16 +2448,13 @@ async fn persist_required_scanner_cycle_floor(
|
||||
state = "invalid_cache_cycle_floor",
|
||||
"Scanner cache cycle floor is invalid"
|
||||
);
|
||||
mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await;
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
return false;
|
||||
}
|
||||
|
||||
cycle_info.next = cycle_info.next.max(required_cycle);
|
||||
cycle_info.current = 0;
|
||||
global_metrics().clear_current_scan_mode();
|
||||
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
persisted
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await
|
||||
}
|
||||
|
||||
async fn await_scanner_cycle_with_lock_fence<Cycle, LockLost>(
|
||||
@@ -2565,7 +2513,7 @@ async fn run_data_scanner_cycle(
|
||||
let now = Instant::now();
|
||||
cycle_info.started = Utc::now();
|
||||
|
||||
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
||||
global_metrics().set_cycle(Some(cycle_info.clone())).await;
|
||||
|
||||
let mut background_heal_info = read_background_heal_info(storeapi.clone()).await;
|
||||
|
||||
@@ -2616,14 +2564,14 @@ async fn run_data_scanner_cycle(
|
||||
"Scanner cycle could not capture the data usage persistence baseline"
|
||||
);
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
};
|
||||
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
||||
let storeapi_clone = storeapi.clone();
|
||||
let ctx_clone = ctx.clone();
|
||||
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
let mut usage_persist_task = tokio::spawn(async move {
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
||||
ctx_clone,
|
||||
storeapi_clone,
|
||||
@@ -2632,9 +2580,10 @@ async fn run_data_scanner_cycle(
|
||||
Some(usage_persist_baseline),
|
||||
)
|
||||
.await
|
||||
}));
|
||||
});
|
||||
|
||||
let done_cycle = Metrics::time(Metric::ScanCycle);
|
||||
let cycle_work_start = global_metrics().start_scan_cycle_work();
|
||||
let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
|
||||
let scan_result = storeapi
|
||||
.clone()
|
||||
@@ -2691,6 +2640,7 @@ async fn run_data_scanner_cycle(
|
||||
}
|
||||
};
|
||||
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
|
||||
global_metrics().finish_scan_cycle_work(cycle_work_start);
|
||||
|
||||
let scan_cycle_result = match scan_result {
|
||||
Ok(result) => result,
|
||||
@@ -2713,7 +2663,7 @@ async fn run_data_scanner_cycle(
|
||||
{
|
||||
save_background_heal_info(storeapi.clone(), new_heal_info).await;
|
||||
}
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
};
|
||||
@@ -2728,7 +2678,7 @@ async fn run_data_scanner_cycle(
|
||||
"Scanner cycle stopped before committing cycle state"
|
||||
);
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
if let Some(required_cycle) = scan_cycle_result.required_cycle_floor() {
|
||||
@@ -2750,7 +2700,6 @@ async fn run_data_scanner_cycle(
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
required_cycle,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2770,7 +2719,7 @@ async fn run_data_scanner_cycle(
|
||||
"Scanner cycle completed without a durable data usage snapshot"
|
||||
);
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
if budget_elapsed {
|
||||
@@ -2794,16 +2743,7 @@ async fn run_data_scanner_cycle(
|
||||
scan_cycle_partial_reason(budget_reason),
|
||||
scan_cycle_partial_source(budget_reason),
|
||||
);
|
||||
return if finalize_partial_scan_cycle(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
return if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
@@ -2852,7 +2792,7 @@ async fn run_data_scanner_cycle(
|
||||
"Scanner cycle completed without a durable data usage snapshot"
|
||||
);
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
ScannerCycleOutcome::Partial => {
|
||||
@@ -2878,16 +2818,7 @@ async fn run_data_scanner_cycle(
|
||||
);
|
||||
}
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
return if finalize_partial_scan_cycle(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
return if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
@@ -2903,16 +2834,7 @@ async fn run_data_scanner_cycle(
|
||||
state = "superseded",
|
||||
"Scanner cycle usage snapshot was superseded by concurrent namespace activity"
|
||||
);
|
||||
if finalize_partial_scan_cycle(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
|
||||
emit_scan_cycle_superseded(cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Superseded;
|
||||
}
|
||||
@@ -2931,7 +2853,7 @@ async fn run_data_scanner_cycle(
|
||||
error = %err,
|
||||
"Scanner completed cycle could not advance"
|
||||
);
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
@@ -2940,8 +2862,9 @@ async fn run_data_scanner_cycle(
|
||||
global_metrics().clear_current_scan_mode();
|
||||
|
||||
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
|
||||
global_metrics().set_cycle(Some(cycle_info.clone())).await;
|
||||
if !persist_scanner_cycle_state(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
mark_scan_cycle_idle(cycle_info).await;
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
@@ -2965,37 +2888,9 @@ async fn run_data_scanner_cycle(
|
||||
"Scanner cycle completed"
|
||||
);
|
||||
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending_maintenance_work)
|
||||
}
|
||||
|
||||
struct ScannerCycleMetricsGuard {
|
||||
start: Option<ScanCycleWorkSnapshot>,
|
||||
}
|
||||
|
||||
impl ScannerCycleMetricsGuard {
|
||||
async fn new(cycle: CurrentCycle) -> Self {
|
||||
Self {
|
||||
start: Some(global_metrics().start_scan_cycle_work_with_cycle(cycle).await),
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish(&mut self, cycle: CurrentCycle) {
|
||||
if let Some(start) = self.start {
|
||||
global_metrics().finish_scan_cycle_work_with_cycle(start, cycle).await;
|
||||
self.start = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScannerCycleMetricsGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(start) = self.start.take() {
|
||||
global_metrics().finish_scan_cycle_work(start);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_scanner_leader_lock_lost(message: &'static str) {
|
||||
reset_scanner_cycle_schedule();
|
||||
record_scanner_leader_lock_state("lost");
|
||||
@@ -3548,7 +3443,7 @@ enum DataUsagePersistTaskResult {
|
||||
|
||||
async fn wait_for_data_usage_persist_task(
|
||||
ctx: &CancellationToken,
|
||||
task: &mut AbortOnDropHandle<DataUsagePersistOutcome>,
|
||||
task: &mut tokio::task::JoinHandle<DataUsagePersistOutcome>,
|
||||
timeout: Duration,
|
||||
) -> DataUsagePersistTaskResult {
|
||||
tokio::select! {
|
||||
@@ -3945,9 +3840,8 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::EcstoreResult;
|
||||
use crate::{
|
||||
Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader,
|
||||
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
|
||||
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
|
||||
ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions,
|
||||
ScannerPutObjReader as PutObjReader,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
@@ -3959,47 +3853,6 @@ mod tests {
|
||||
|
||||
const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
|
||||
|
||||
async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let disk_path = temp_dir.path().join(format!("disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("scanner cycle test disk should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "scanner-cycle-metrics".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
}]);
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
.await
|
||||
.expect("scanner cycle test disks should initialize");
|
||||
let store = ECStore::new_with_instance_ctx(
|
||||
"127.0.0.1:0".parse().expect("test address should parse"),
|
||||
endpoint_pools,
|
||||
CancellationToken::new(),
|
||||
instance_ctx,
|
||||
)
|
||||
.await
|
||||
.expect("scanner cycle test ECStore should initialize");
|
||||
init_bucket_metadata_sys_for_scanner_tests(store.clone()).await;
|
||||
|
||||
(temp_dir, store)
|
||||
}
|
||||
|
||||
fn assert_run_data_scanner_signature<F, Fut>(_run: F)
|
||||
where
|
||||
F: Fn(CancellationToken, Arc<ECStore>) -> Fut,
|
||||
@@ -4459,9 +4312,9 @@ mod tests {
|
||||
};
|
||||
|
||||
global_metrics().set_current_scan_mode(HealScanMode::Deep);
|
||||
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
||||
global_metrics().set_cycle(Some(cycle_info.clone())).await;
|
||||
|
||||
mark_scan_cycle_idle(&mut cycle_info, &mut cycle_metrics_guard).await;
|
||||
mark_scan_cycle_idle(&mut cycle_info).await;
|
||||
|
||||
let published = global_metrics()
|
||||
.get_cycle()
|
||||
@@ -4477,123 +4330,6 @@ mod tests {
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let mut cycle_info = CurrentCycle {
|
||||
current: 0,
|
||||
next: 1,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
||||
let setup_report = global_metrics().report().await;
|
||||
assert!(setup_report.current_cycle_active);
|
||||
assert_eq!(setup_report.current_cycle, 0);
|
||||
assert_eq!(setup_report.current_started, cycle_started);
|
||||
|
||||
mark_scan_cycle_idle(&mut cycle_info, &mut guard).await;
|
||||
let idle_report = global_metrics().report().await;
|
||||
assert!(!idle_report.current_cycle_active);
|
||||
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() {
|
||||
let mut cycle_info = CurrentCycle {
|
||||
current: 12,
|
||||
next: 13,
|
||||
started: Utc::now(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
||||
|
||||
cycle_info.current = 0;
|
||||
tokio::task::yield_now().await;
|
||||
let finalizing_report = global_metrics().report().await;
|
||||
assert!(finalizing_report.current_cycle_active);
|
||||
assert_eq!(finalizing_report.current_cycle, 12);
|
||||
|
||||
guard.finish(cycle_info).await;
|
||||
let idle_report = global_metrics().report().await;
|
||||
assert!(!idle_report.current_cycle_active);
|
||||
assert_eq!(idle_report.current_cycle, 0);
|
||||
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_metrics_guard_drop_clears_activity() {
|
||||
let guard = ScannerCycleMetricsGuard::new(CurrentCycle {
|
||||
current: 12,
|
||||
next: 13,
|
||||
started: Utc::now(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(global_metrics().report().await.current_cycle_active);
|
||||
|
||||
drop(guard);
|
||||
|
||||
assert!(!global_metrics().report().await.current_cycle_active);
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
let ctx = CancellationToken::new();
|
||||
let mut cycle_info = CurrentCycle::default();
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let leader_epoch = u64::MAX - 1;
|
||||
let state_persist_reached = Arc::new(Notify::new());
|
||||
let _state_persist_hook = set_scanner_cycle_state_persist_test_hook(leader_epoch, state_persist_reached.clone());
|
||||
let state_lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("scanner cycle state lock should be created");
|
||||
let state_guard = state_lock
|
||||
.get_write_lock(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("scanner cycle state lock should be acquired");
|
||||
let mut cycle = Box::pin(run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch));
|
||||
let waker = std::task::Waker::noop();
|
||||
let mut context = std::task::Context::from_waker(waker);
|
||||
|
||||
assert!(cycle.as_mut().poll(&mut context).is_pending());
|
||||
let active = global_metrics().report().await;
|
||||
assert!(active.current_cycle_active);
|
||||
assert_eq!(active.current_cycle, 0);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
tokio::select! {
|
||||
outcome = &mut cycle => panic!("scanner cycle finished before state persistence was released: {outcome:?}"),
|
||||
_ = state_persist_reached.notified() => {}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("scanner cycle should reach state persistence");
|
||||
let finalizing = global_metrics().report().await;
|
||||
assert!(finalizing.current_cycle_active);
|
||||
|
||||
drop(state_guard);
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(30), cycle)
|
||||
.await
|
||||
.expect("scanner cycle should finish");
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
|
||||
));
|
||||
assert!(!global_metrics().report().await.current_cycle_active);
|
||||
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
|
||||
@@ -4606,11 +4342,8 @@ mod tests {
|
||||
cycle_completed: vec![],
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
||||
|
||||
assert!(
|
||||
finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await
|
||||
);
|
||||
assert!(finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1).await);
|
||||
|
||||
assert_eq!(cycle_info.next, 13);
|
||||
assert_eq!(cycle_info.current, 0);
|
||||
@@ -4644,20 +4377,8 @@ mod tests {
|
||||
cycle_completed: vec![],
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
||||
|
||||
assert!(
|
||||
persist_required_scanner_cycle_floor(
|
||||
&ctx,
|
||||
store.clone(),
|
||||
&mut cycle_info,
|
||||
&mut revision,
|
||||
7,
|
||||
19,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
);
|
||||
assert!(persist_required_scanner_cycle_floor(&ctx, store.clone(), &mut cycle_info, &mut revision, 7, 19).await);
|
||||
assert_eq!(cycle_info.current, 0);
|
||||
assert_eq!(cycle_info.next, 19);
|
||||
|
||||
@@ -4684,37 +4405,22 @@ mod tests {
|
||||
cycle_completed: vec![],
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
||||
|
||||
assert!(
|
||||
!persist_required_scanner_cycle_floor(
|
||||
&ctx,
|
||||
store.clone(),
|
||||
&mut cycle_info,
|
||||
&mut revision,
|
||||
7,
|
||||
12,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
);
|
||||
assert!(!persist_required_scanner_cycle_floor(&ctx, store.clone(), &mut cycle_info, &mut revision, 7, 12).await);
|
||||
assert_eq!(cycle_info.next, 12);
|
||||
assert_eq!(revision, DataUsageCacheRevision::Missing);
|
||||
let mut max_cycle_info = CurrentCycle {
|
||||
current: 12,
|
||||
next: 12,
|
||||
..Default::default()
|
||||
};
|
||||
let mut max_cycle_metrics_guard = ScannerCycleMetricsGuard::new(max_cycle_info.clone()).await;
|
||||
assert!(
|
||||
!persist_required_scanner_cycle_floor(
|
||||
&ctx,
|
||||
store.clone(),
|
||||
&mut max_cycle_info,
|
||||
&mut CurrentCycle {
|
||||
current: 12,
|
||||
next: 12,
|
||||
..Default::default()
|
||||
},
|
||||
&mut revision,
|
||||
7,
|
||||
u64::MAX,
|
||||
&mut max_cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
);
|
||||
@@ -4979,9 +4685,8 @@ mod tests {
|
||||
cycle_completed: vec![],
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
||||
|
||||
assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await);
|
||||
assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1).await);
|
||||
assert_eq!(cycle_info.next, 13);
|
||||
assert_eq!(cycle_info.current, 0);
|
||||
assert_eq!(revision, DataUsageCacheRevision::Missing);
|
||||
@@ -6238,10 +5943,10 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() {
|
||||
let ctx = CancellationToken::new();
|
||||
let mut task = AbortOnDropHandle::new(tokio::spawn(async {
|
||||
let mut task = tokio::spawn(async {
|
||||
std::future::pending::<()>().await;
|
||||
DataUsagePersistOutcome::Saved
|
||||
}));
|
||||
});
|
||||
ctx.cancel();
|
||||
|
||||
let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(60)).await;
|
||||
@@ -6253,10 +5958,10 @@ mod tests {
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn data_usage_persist_wait_aborts_after_timeout() {
|
||||
let ctx = CancellationToken::new();
|
||||
let mut task = AbortOnDropHandle::new(tokio::spawn(async {
|
||||
let mut task = tokio::spawn(async {
|
||||
std::future::pending::<()>().await;
|
||||
DataUsagePersistOutcome::Saved
|
||||
}));
|
||||
});
|
||||
|
||||
let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(30)).await;
|
||||
|
||||
|
||||
@@ -58,16 +58,11 @@ url = { workspace = true, optional = true }
|
||||
zstd = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
temp-env = { workspace = true }
|
||||
proptest = "1"
|
||||
|
||||
[[bench]]
|
||||
name = "hash_hotpath_benchmark"
|
||||
harness = false
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows = { workspace = true, optional = true, features = ["Win32_Storage_FileSystem"] }
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user